{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true,
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "## 用TensorFlow并行训练神经网络\n",
    "pip install tensorflow -i https://pypi.tuna.tsinghua.edu.cn/simple"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "outputs": [
    {
     "data": {
      "text/plain": "'2.8.0'"
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import tensorflow.compat.v1 as tf  #使用1.0版本的tensorflow\n",
    "\n",
    "#tf.disable_v2_behavior()\n",
    "#tf.compat.v1.disable_eager_execution()\n",
    "\n",
    "tf.__version__"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "## 底层API的使用"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "x= 1.0 -->z= 2.7\n",
      "x= 0.6 -->z= 1.9\n",
      "x=-1.8 -->z=-2.9\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-05-08 14:32:31.139786: I tensorflow/core/platform/cpu_feature_guard.cc:151] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA\n",
      "To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n"
     ]
    }
   ],
   "source": [
    "#计算z=w*x+b\n",
    "g = tf.Graph()\n",
    "#创建一个计算图\n",
    "with g.as_default():\n",
    "    #创建一个变量占位符\n",
    "    x = tf.placeholder(dtype=tf.float32,\n",
    "                       shape=(None),\n",
    "                       name='x')\n",
    "    w = tf.Variable(2.0, name='weight')\n",
    "    b = tf.Variable(0.7, name='bias')\n",
    "    z = w * x + b\n",
    "    init = tf.global_variables_initializer()  #含有tf.Variable的环境下，因为tf中建立的变量是没有初始化的，也就是在debug时还不是一个tensor量，而是一个Variable变量类型\n",
    "with tf.Session(graph=g) as sess:\n",
    "    sess.run(init)  #初始化计算图一遍\n",
    "    for t in [1.0, 0.6, -1.8]:\n",
    "        print('x=%4.1f -->z=%4.1f' % (t, sess.run(z, feed_dict={x: t})))  #传入对应的x对应的值"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[[ 0  1  2]\n",
      "  [ 3  4  5]]\n",
      "\n",
      " [[ 6  7  8]\n",
      "  [ 9 10 11]]\n",
      "\n",
      " [[12 13 14]\n",
      "  [15 16 17]]]\n",
      "input shape:  (3, 2, 3)\n",
      "Reshaped:\n",
      " [[ 0.  1.  2.  3.  4.  5.]\n",
      " [ 6.  7.  8.  9. 10. 11.]\n",
      " [12. 13. 14. 15. 16. 17.]]\n",
      "Column Sums:\n",
      " [18. 21. 24. 27. 30. 33.]\n",
      "Column Means:\n",
      " [ 6.  7.  8.  9. 10. 11.]\n"
     ]
    }
   ],
   "source": [
    "#计算阵列结构的张量\n",
    "import numpy as np\n",
    "\n",
    "g = tf.Graph()\n",
    "with g.as_default():\n",
    "    x = tf.placeholder(dtype=tf.float32,\n",
    "                       shape=(None, 2, 3),\n",
    "                       name=\"input_x\")\n",
    "    x2 = tf.reshape(x, shape=(-1, 6),  #数组转换为6列\n",
    "                    name=\"x2\")\n",
    "    xsum = tf.reduce_sum(x2, axis=0, name='col_sum')  #计算列和\n",
    "    xmean = tf.reduce_mean(x2, axis=0, name='col_mean')  #计算列均值\n",
    "with tf.Session(graph=g) as sess:\n",
    "    x_array = np.arange(18).reshape(3, 2, 3)\n",
    "    print(x_array)\n",
    "    print('input shape: ', x_array.shape)\n",
    "    print('Reshaped:\\n',\n",
    "          sess.run(x2, feed_dict={x: x_array}))\n",
    "    print('Column Sums:\\n',\n",
    "          sess.run(xsum, feed_dict={x: x_array}))\n",
    "    print('Column Means:\\n',\n",
    "          sess.run(xmean, feed_dict={x: x_array}))"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "## 使用底层API实现最小二乘估计"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "outputs": [],
   "source": [
    "X_train = np.arange(10).reshape((10, 1))\n",
    "y_train = np.array([1, 1.3, 3.1, 2, 5, 6.3, 6.6, 7.4, 8, 9])\n",
    "\n",
    "\n",
    "class TfLinreg(object):\n",
    "    def __init__(self, x_dim, learning_rate=1.0, random_seed=None):\n",
    "        self.x_dim = x_dim\n",
    "        self.learning_rate = learning_rate\n",
    "        self.g = tf.Graph()\n",
    "        #创建一个计算图\n",
    "        with self.g.as_default():\n",
    "            tf.set_random_seed(random_seed)\n",
    "            self.build()\n",
    "            self.init_op = tf.global_variables_initializer()\n",
    "\n",
    "    def build(self):\n",
    "        self.X = tf.placeholder(dtype=tf.float32,\n",
    "                                shape=(None, self.x_dim),\n",
    "                                name='x_input')\n",
    "        self.y = tf.placeholder(dtype=tf.float32,\n",
    "                                shape=(None),\n",
    "                                name='y_input')\n",
    "        print(self.X)\n",
    "        print(self.y)\n",
    "        w = tf.Variable(tf.zeros(shape=(1)),\n",
    "                        name='weight')\n",
    "        b = tf.Variable(tf.zeros(shape=(1)),\n",
    "                        name=\"bias\")\n",
    "        print(w)\n",
    "        print(b)\n",
    "        self.z_net = tf.squeeze(w * self.X + b,  #删除维度为1的张量\n",
    "                                name='z_net')\n",
    "        print(self.z_net)\n",
    "        sqr_errors = tf.square(self.y - self.z_net,\n",
    "                               name='sqr_errors')\n",
    "        print(sqr_errors)\n",
    "        self.mean_cost = tf.reduce_mean(sqr_errors,\n",
    "                                        name='mean_cost')\n",
    "        optimizer = tf.train.GradientDescentOptimizer(  #使用梯度下降\n",
    "            learning_rate=self.learning_rate,\n",
    "            name='GradientDescent'\n",
    "        )\n",
    "        self.optimizer = optimizer.minimize(self.mean_cost)  #计算梯度，并自动更新参数"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tensor(\"x_input:0\", shape=(None, 1), dtype=float32)\n",
      "Tensor(\"y_input:0\", dtype=float32)\n",
      "<tf.Variable 'weight:0' shape=(1,) dtype=float32>\n",
      "<tf.Variable 'bias:0' shape=(1,) dtype=float32>\n",
      "Tensor(\"z_net:0\", dtype=float32)\n",
      "Tensor(\"sqr_errors:0\", dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "lrmodel = TfLinreg(x_dim=X_train.shape[1], learning_rate=0.01)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "outputs": [],
   "source": [
    "def train_linreg(sess, model, X_train, y_train, num_epochs=10):\n",
    "    #初始化所有的变量w和b\n",
    "    sess.run(model.init_op)\n",
    "    training_costs = []\n",
    "    for i in range(num_epochs):\n",
    "        _, cost = sess.run([model.optimizer, model.mean_cost],\n",
    "                           feed_dict={model.X: X_train,\n",
    "                                      model.y: y_train})\n",
    "        training_costs.append(cost)\n",
    "    return training_costs"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "outputs": [],
   "source": [
    "sess = tf.Session(graph=lrmodel.g)\n",
    "training_costs = train_linreg(sess, lrmodel, X_train, y_train)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "outputs": [
    {
     "data": {
      "text/plain": "<Figure size 432x288 with 1 Axes>",
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAbYAAAEmCAYAAAAOb7UzAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAe9ElEQVR4nO3de5RcZZnv8d9T1d3pW1KVpDuYdBVpbnIJpBppEYjLhSJncZw5Bsbj9YA4OqIOjDDjmlFn1ppxzjmjnpnjhRkZNCqKysBwBAVRUUDUxUWgkU4IJMotkJBO0mlMutOxu1Ndz/mjdscm105Tu966fD9r1apdb+3a++ni8qu933e/29xdAADUikToAgAAKCWCDQBQUwg2AEBNIdgAADWFYAMA1JSG0AXMREdHh3d3d4cuAwBQQR599NHt7t65b3tVBFt3d7f6+vpClwEAqCBm9vyB2jkVCQCoKQQbAKCmEGwAgJpCsAEAagrBBgCoKQQbAKCmEGwAgJpCsAEAakrdBNtEvqDfT0yGLgMAELO6CLahXeM69VM/0c19G0OXAgCIWV0E24K2JqVbGrV6447QpQAAYlYXwWZmymXTWr1pR+hSAAAxq4tgk6SebFrPDI5qeGxP6FIAADGqm2BbnklJkh7ftDNwJQCAONVPsHWlJUn99LMBQE2rm2BLtTbq2I42BpAAQI2rm2CTxAASAKgD9RVsmZS2Do9ry86x0KUAAGJSX8GWTUuinw0AalldBdvJi+epMWlaw+lIAKhZdRVszY1Jnbx4Hv1sAFDD6irYpOL1bGs27lSh4KFLAQDEoO6CLZdJa2Q8r2e3j4YuBQAQg7oLtp5oAAnXswFAbYot2Mys2cweNrPVZvaEmf1j1L7AzO4ys6ei5/lx1XAgx3a2q31OA/1sAFCj4jxiG5f0JnfPSeqRdIGZnSXpE5LucfcTJN0TvS6bZMJ0WleKIzYAqFGxBZsX7YpeNkYPl7RS0vVR+/WSLoyrhoPJZdN6cmBY43nuqA0AtSbWPjYzS5pZv6Rtku5y94ckHeXuA5IUPS86yGcvM7M+M+sbHBwsaV092ZT2TLrWD4yUdLsAgPBiDTZ3n3T3HkkZSWea2alH8NlV7t7r7r2dnZ0lrWtqBhL62QCg9pRlVKS775D0c0kXSNpqZoslKXreVo4apnvVvGZ1zp3D1FoAUIPiHBXZaWbpaLlF0pslrZd0u6RLo9UulXRbXDUcojblMmkGkABADWqIcduLJV1vZkkVA/Rmd7/DzB6UdLOZfUDSC5LeHmMNB9WTTenudVs1PLZH85obQ5QAAIhBbMHm7msknX6A9iFJ58W135ma6md7fNNOrTi+I2wxAICSqbuZR6Ys70pL4hY2AFBr6jbYUq2NOrajjX42AKgxdRtsUvF05JpNO0OXAQAooboOtuWZlLYMj2nLzrHQpQAASqSug40LtQGg9tR1sJ2yeJ4aEkY/GwDUkLoOtubGpE5ePI8jNgCoIXUdbJKUy6a0ZuNOFQoeuhQAQAkQbJm0Rsbzenb7aOhSAAAlUPfB1jM1gIR+NgCoCXUfbMd2tqt9ToPW0M8GADWh7oMtmTCd2jVP/VyoDQA1oe6DTSpez7Zu87DG85OhSwEAvEIEm6SeTFoTkwWtHxgJXQoA4BUi2MQMJABQSwg2SYtTzeqcO4db2ABADSDYJJmZcpk0Q/4BoAYQbJGebErPDI5qeGxP6FIAAK8AwRaZ6mdby7B/AKhqBFtkeVdaktTPABIAqGoEWyTV2qhjOtroZwOAKkewTZPLpLR6I6ciAaCaEWzT5LJpbRke05adY6FLAQDMEsE2DRdqA0D1I9imOWXxPDUkjH42AKhiBNs0zY1Jnbx4HkdsAFDFYgs2M8ua2b1mts7MnjCzK6P2T5nZi2bWHz3eElcNs5HLprRm004VCh66FADALMR5xJaX9DF3P1nSWZIuN7NTove+4O490eNHMdZwxJZn0hoZy+u5odHQpQAAZiG2YHP3AXf/dbQ8ImmdpK649lcqPVMDSOhnA4CqVJY+NjPrlnS6pIeipivMbI2ZXWdm8w/ymcvMrM/M+gYHB8tRpiTpuM52tTUlCTYAqFKxB5uZtUu6RdJV7j4s6VpJx0nqkTQg6XMH+py7r3L3Xnfv7ezsjLvMvZIJ02mZlPqZMxIAqlKswWZmjSqG2g3ufqskuftWd59094Kkr0o6M84aZiOXTWvd5mGN5ydDlwIAOEJxjoo0SV+XtM7dPz+tffG01S6StDauGmarJ5PWxGRB6wdGQpcCADhCDTFue4WkSyQ9bmb9UdvfSnq3mfVIckkbJH0oxhpmZfoMJFPLAIDqEFuwuft9kuwAb1XU8P4DWZxqVkf7nOKEyGeHrgYAcCSYeeQAzEw92RQzkABAFSLYDiKXSeuZwV0aHtsTuhQAwBEg2A4il03LXVrLsH8AqCoE20Esz6QkSf2cjgSAqkKwHUS6tUnHdLQxAwkAVBmC7RBymVRxZCQAoGoQbIeQy6a1ZXhMW3aOhS4FADBDBNshLM+kJYlh/wBQRQi2Q1i2ZJ4aEqY1BBsAVA2C7RCaG5M6afFc+tkAoIoQbIeRy6S1etMOFQoeuhQAwAwQbIeRy6Y1MpbXc0OjoUsBAMwAwXYYPVMz/XM9GwBUBYLtMI7rbFdbU5JgA4AqQbAdRjJhOi2TUj9zRgJAVSDYZiCXSWvd5mFN5AuhSwEAHAbBNgO5bFoTkwWt3zIcuhQAwGEQbDOQYwAJAFQNgm0GlqSa1dE+R/1cqA0AFY9gmwEzU082xZyRAFAFCLYZymXSemZwl4bH9oQuBQBwCATbDOWyablLaxn2DwAVjWCboeWZlCSpn9ORAFDRCLYZSrc2qXthKyMjAaDCEWxHIJdNaw2nIgGgohFsRyCXSWtg55i2Do+FLgUAcBAE2xHgQm0AqHyxBZuZZc3sXjNbZ2ZPmNmVUfsCM7vLzJ6KnufHVUOpLVsyTw0J43o2AKhgcR6x5SV9zN1PlnSWpMvN7BRJn5B0j7ufIOme6HVVaG5M6qTFc7WaGUgAoGLFFmzuPuDuv46WRyStk9QlaaWk66PVrpd0YVw1xCGXSWv1ph0qFDx0KQCAAyhLH5uZdUs6XdJDko5y9wGpGH6SFh3kM5eZWZ+Z9Q0ODpajzBnJZdIaGcvruaHR0KUAAA4g9mAzs3ZJt0i6yt1nfN8Xd1/l7r3u3tvZ2RlfgUeIASQAUNliDTYza1Qx1G5w91uj5q1mtjh6f7GkbXHWUGrHL2pXa1OS69kAoELFOSrSJH1d0jp3//y0t26XdGm0fKmk2+KqIQ7JhOm0rpT6OWIDgIoU5xHbCkmXSHqTmfVHj7dI+qyk883sKUnnR6+rSk82rSc3D2siXwhdCgBgHw1xbdjd75NkB3n7vLj2Ww65bFoTkwWt3zKs5Zl06HIAANMw88gsMIAEACoXwTYLS1LN6mifo34u1AaAikOwzYKZKZdJMbUWAFQggm2Wctm0nhncpZGxPaFLAQBMQ7DNUi6blrv0+IucjgSASkKwzVIuk5IkJkQGgApDsM1SurVJ3QtbGRkJABWGYHsFctk0A0gAoMIQbK9ALpPWwM4xbR0eC10KACBCsL0CXKgNAJWHYHsFli2Zp4aEcToSACrIjILNzL49k7Z609yY1ImvmsvISACoIDM9Yls2/YWZJSWdUfpyqk8um9aaTTtUKHjoUgAAOkywmdknzWxE0nIzG44eIyreHLSq7qMWl55MWsNjeW0YGg1dCgBAhwk2d/+Mu8+V9C/uPi96zHX3he7+yTLVWNH2DiChnw0AKsJMT0XeYWZtkmRmF5vZ581saYx1VY3jF7WrtSlJPxsAVIiZBtu1knabWU7S30h6XtK3YquqiiQTptO6UupnyD8AVISZBlve3V3SSklXu/vVkubGV1Z16cmm9eTmYU3kC6FLAYC6N9NgGzGzT0q6RNIPo1GRjfGVVV2WZ9KamCxo/Zbh0KUAQN2babC9U9K4pPe7+xZJXZL+JbaqqkwuOzXT/46whQAAZhZsUZjdICllZn8saczd6WOLdKVb1NHepNWbGEACAKHNdOaRd0h6WNLbJb1D0kNm9t/jLKyamJlymTRHbABQARpmuN7fSXqtu2+TJDPrlHS3pO/GVVi1yWXT+tlvtmlkbI/mNtP9CAChzLSPLTEVapGhI/hsXchl03KXHn+R05EAENJMw+lOM/uJmb3PzN4n6YeSfhRfWdUnl5kaQEKwAUBIhzwVaWbHSzrK3f/azP5E0uslmaQHVRxMgki6tUlLF7bSzwYAgR3uiO2LkkYkyd1vdfe/cve/VPFo7YuH+qCZXWdm28xs7bS2T5nZi2bWHz3e8srKryy5TJo5IwEgsMMFW7e7r9m30d37JHUf5rPflHTBAdq/4O490aOmTmfmsmkN7BzTtuGx0KUAQN06XLA1H+K9lkN90N1/KemlI66oivVMXajN9WwAEMzhgu0RM/vgvo1m9gFJj85yn1eY2ZroVOX8g61kZpeZWZ+Z9Q0ODs5yV+W1bElKyYTRzwYAAR0u2K6S9Kdm9nMz+1z0+IWkP5N05Sz2d62k4yT1SBqQ9LmDrejuq9y91917Ozs7Z7Gr8mtuTOqkV82lnw0AAjrkqEh33yrpHDN7o6RTo+YfuvvPZrOzaHuSJDP7qqQ7ZrOdSpbLpnXH6s0qFFyJhIUuBwDqzkznirzX3f8teswq1CTJzBZPe3mRpLUHW7da9WTSGh7La8PQaOhSAKAuzXRKrSNmZjdKOldSh5ltkvQPks41sx5JLmmDpA/Ftf9Qlu8dQLJDx3a2B64GAOpPbMHm7u8+QPPX49pfpThh0Vy1NiW1euNOXXR6JnQ5AFB3mO+xxJIJ06ldKfUzMhIAgiDYYtCTTevJgWFN5AuhSwGAukOwxSCXSWsiX9BvtoyELgUA6g7BFoNcNICkn+vZAKDsCLYYdKVb1NHexAwkABAAwRYDMyvO9E+wAUDZEWwxWZ5J6+nBXRoZ2xO6FACoKwRbTHLZlNylx19kpn8AKCeCLSa5TFqStHojwQYA5USwxWR+W5OWLmzVGkZGAkBZEWwxYgAJAJQfwRajXDatzTvHtG14LHQpAFA3CLYY9eyd6Z9+NgAoF4ItRsuWpJRMGKcjAaCMCLYYNTcmdeJRc7WaASQAUDYEW8xy2eIAkkLBQ5cCAHWBYItZTzal4bG8NgyNhi4FAOoCwRazXDYtSVrDABIAKAuCLWYnLJqr1qYkd9QGgDIh2GKWTJhO7UoxgAQAyoRgK4OebFpPbB7WRL4QuhQAqHkEWxksz6Q0kS/oN1tGQpcCADWPYCuDqZn++zkdCQCxI9jKIDO/RQvbmpiBBADKgGArAzPbe6E2ACBeBFuZ5DJpPT24S7vG86FLAYCaRrCVSS6bkrv0OBdqA0CsYgs2M7vOzLaZ2dppbQvM7C4zeyp6nh/X/ivN1AASrmcDgHjFecT2TUkX7NP2CUn3uPsJku6JXteF+W1NWrqwlX42AIhZbMHm7r+U9NI+zSslXR8tXy/pwrj2X4mWZxhAAgBxK3cf21HuPiBJ0fOig61oZpeZWZ+Z9Q0ODpatwDjlMilt3jmmbcNjoUsBgJpVsYNH3H2Vu/e6e29nZ2fockqiJ5rpfzUDSAAgNuUOtq1mtliSoudtZd5/UMuWpJRMGKcjASBG5Q622yVdGi1fKum2Mu8/qJampE48ai4jIwEgRnEO979R0oOSTjSzTWb2AUmflXS+mT0l6fzodV2ZmoHE3UOXAgA1qSGuDbv7uw/y1nlx7bMa9GRTuvHhF7RhaLeO6WgLXQ4A1JyKHTxSq3JTA0joZwOAWBBsZXZ8Z7taGpPqJ9gAIBYEW5k1JBM6rSvFABIAiAnBFkAum9ITm4c1kS+ELgUAag7BFkAum9ZEvqDfbBkJXQoA1ByCLQBm+geA+BBsAWTmt2hhWxMjIwEgBgRbAGZWvFCbIzYAKDmCLZBcJq2ntu3SrvF86FIAoKYQbIEsz6bkLj3OTP8AUFIEWyAMIAGAeBBsgSxoa9LRC1oZQAIAJUawBTQ10z8AoHQItoBymZQ27xzTtpGx0KUAQM0g2ALqiWb6X7ORASQAUCoEW0DLlqSUTBgDSACghAi2gFqaknr1UXO5hQ0AlBDBFlhPNqXVG3fI3UOXAgA1gWALLJdJa3gsrw1Du0OXAgA1gWALLBcNIGHYPwCUBsEW2AmL2tXSmKSfDQBKhGALrCGZ0GldKT32wu/oZwOAEiDYKsC5J3Vq9aad+tj/W62xPZOhywGAqtYQugBIH37DcdqTd33h7t/que2j+srFZ2jRvObQZQFAVeKIrQIkEqYr33yCvnzxa7R+YERv/dL9WsNF2wAwKwRbBbng1MW65SPnKJkwvf3LD+q2/hdDlwQAVSdIsJnZBjN73Mz6zawvRA2V6pQl83TbFSuUy6R15U39+uc716tQYFAJAMxUyCO2N7p7j7v3BqyhInW0z9F3/ux1eveZWf37z5/RZd/u08jYntBlAUBV4FRkhWpqSOjTF52m/7lyme79zaDedu0Den5oNHRZAFDxQgWbS/qpmT1qZpcdaAUzu8zM+sysb3BwsMzlVQYz03vP7ta33n+mtg6Pa+U19+uBp7eHLgsAKlqoYFvh7q+R9F8lXW5mb9h3BXdf5e697t7b2dlZ/goryIrjO3T7FSvU2T5Hl1z3sL714AYu5gaAgwgSbO6+OXreJul7ks4MUUc1WbqwTbf++Tk699Wd+vvbntDffX+tJvKF0GUBQMUpe7CZWZuZzZ1alvRfJK0tdx3VaG5zo1a9t1cfOfc4/cdDL+jirz+koV3jocsCgIoS4ojtKEn3mdlqSQ9L+qG73xmgjqqUTJg+fsFJuvpdPVq9cYfe+qX7tW5gOHRZAFAxyh5s7v6su+eixzJ3/6dy11ALVvZ06eYPna18oaC3XfuA7lw7ELokAKgIDPevYrlsWj+44vV69VFz9eHv/FpX3/0Ug0oA1D2Crcotmtesmy47S39yepe+cPdvdcV/PKbdE/nQZQFAMMzuXwOaG5P63DtyOmnxXH3mx+v13PZRffXSXnWlW0KXBgBlxxFbjTAzXfaG43Tdpa/Vxpd2663/dp8e2fBS6LIAoOwIthrzxpMW6XuXr9C8lka956u/0n8+8kLokgCgrAi2GnT8onZ9/89X6KxjF+rjtzyuf/zBE8pPcjE3gPpAsNWoVGujvvG+1+r9K47RN+7foPd94xHt2D0RuiwAiB3BVsMakgn9/X87Rf/8tuV66LkhXXjN/Xp620josgAgVgRbHXjHa7O68YNnadd4Xhde84DuXb8tdEkAEBuCrU70di/QbVe8XkcvaNX7r39EX/nFM1zMDaAmEWx1pCvdou9+5Gy95dTF+syP1+uvbl6tsT2TocsCgJIi2OpMa1ODvvSe0/Wx81+t7z32ot656lfaOjwWuiwAKBmCrQ6Zmf7ivBP05YvP0FNbR/TWL92n1Rt3hC4LAEqCYKtjF5z6Kt3ykXPUkEjo7V95UN9/7MXQJQHAK0aw1bmTF8/T7VesUE82rav+s1+f/fF6TRYYVAKgehFs0ML2OfrOB16n97zuaH35F8/og9/q08jYntBlAcCsEGyQJDU1JPTpi07T/1q5TL/47aAu+vcHtGH7aOiyAOCIEWx4mUvO7ta333+mtu8a18pr7tf9T28PXRIAHBGCDfs55/gO3X7563XUvDl673UP6xv3P0e/G4CqYdUw+0Rvb6/39fWFLqPu7BrP66qb+nX3uq1qTJqyC1rVvbCt+Oho1dKFbTpmYZuWpJvVkOQ3EoDyMrNH3b1333buoI2Dap/ToFWXnKEfrNms9VtGtGH7qDYM7davnh3S7ok/zFjSmDRl57dq6cIo7DratHRhMQQz81sIPQBlRbDhkBIJ08qeLq2c1ubuGhwZ14ah3VHYjer5od16bvuoHn7uJY1OC72GhCkzv2W/wOvuKIZeI6EHoMQINhwxM9Oiec1aNK9ZZx6z4GXvubu275rQhqHRvaG3YWi3nh8a1aPP/067xvN7100mTF3pFnV3tKl779Fe8Tk7v1VNDYQegCNHsKGkzEydc+eoc+4cvbZ7/9AbGp3Q80Ojem57Meymjvoee/53GpkWegmTuua3qHvhtKO8qG8vu6BVcxqS5f7TAFQJgg1lY2bqaJ+jjvY5OmPp/qH30ujE3qO7qf6854dGdXv/Zg2P5adtR1qSalF3RzHwFs1tVlNDQo1J05yGhBqT0aMhoaZkQk0NpsZkcfkPbVPrmZqitsaovSFhMrNyfz0ASoRgQ0UwMy1sn6OF7XN0xtL5L3vP3bVj957otOaoNmzfvfcU5x1rBrTz96WfJeUP4Wd7A69pb2Day4Kw6RBB2pBMKGHFMDaZEibJTNGTTBY9F/szdYB2MylhU+9N/2zx9SG3GS3Yvp+dtt6B/lm87PUBvp99P2f7rHXA7R5mGwdaaza/L2bzk2Q2P2RK/dOn1L+lSr69Ev7FS9ItOvFVc0u2vX0RbKh4Zqb5bU2a39ak04+ev9/7+cmC9ky6JiYL2jNZ0ES++LxnsqDxfPG9PZMF7ckXNB49F9ef1J588XPTPzORL2hi6jN7X0efyU/u3d54vqBd4/lp+/Rp6/5hPwV3uYoBXXwu+1cIVJT3vO5offqi02LbfpBgM7MLJF0tKSnpa+7+2RB1oDY0JBNqSEotqp5+N3eXu/YLvEKUesX3Xr5OwSUdoN0VfW5v28vXKRQOvs19M3b/0N0/hfdd5/DbKO73sOvst90j/wVQrh8Npd7PbP7WQ26v5PWV1sK2phJv8eXKHmxmlpR0jaTzJW2S9IiZ3e7uT5a7FiAUs+mnAenPA0opxHjqMyU97e7PuvuEpJukl10mBQDArIUIti5JG6e93hS1vYyZXWZmfWbWNzg4WLbiAADVLUSwHei8y/5n9t1XuXuvu/d2dnaWoSwAQC0IEWybJGWnvc5I2hygDgBADQoRbI9IOsHMjjGzJknvknR7gDoAADWo7KMi3T1vZldI+omKw/2vc/cnyl0HAKA2BbmOzd1/JOlHIfYNAKhtTJ8OAKgpBBsAoKaYV8HEdWY2KOn50HXErEPS9tBFVCm+u9nju5s9vrvZK9V3t9Td97serCqCrR6YWZ+794auoxrx3c0e393s8d3NXtzfHaciAQA1hWADANQUgq1yrApdQBXju5s9vrvZ47ubvVi/O/rYAAA1hSM2AEBNIdgAADWFYAvMzLJmdq+ZrTOzJ8zsytA1VRMzS5rZY2Z2R+haqo2Zpc3su2a2Pvr37+zQNVUDM/vL6L/VtWZ2o5k1h66pkpnZdWa2zczWTmtbYGZ3mdlT0fP8Uu6TYAsvL+lj7n6ypLMkXW5mpwSuqZpcKWld6CKq1NWS7nT3kyTlxPd4WGbWJemjknrd/VQVJ3J/V9iqKt43JV2wT9snJN3j7idIuid6XTIEW2DuPuDuv46WR1T8n8t+dxTH/swsI+mPJH0tdC3VxszmSXqDpK9LkrtPuPuOoEVVjwZJLWbWIKlV3E/ykNz9l5Je2qd5paTro+XrJV1Yyn0SbBXEzLolnS7pocClVIsvSvobSYXAdVSjYyUNSvpGdCr3a2bWFrqoSufuL0r6v5JekDQgaae7/zRsVVXpKHcfkIo/7iUtKuXGCbYKYWbtkm6RdJW7D4eup9KZ2R9L2ubuj4aupUo1SHqNpGvd/XRJoyrx6aBaFPUFrZR0jKQlktrM7OKwVWFfBFsFMLNGFUPtBne/NXQ9VWKFpLea2QZJN0l6k5l9J2xJVWWTpE3uPnV24LsqBh0O7c2SnnP3QXffI+lWSecErqkabTWzxZIUPW8r5cYJtsDMzFTs51jn7p8PXU+1cPdPunvG3btV7Lz/mbvzy3mG3H2LpI1mdmLUdJ6kJwOWVC1ekHSWmbVG/+2eJwbdzMbtki6Nli+VdFspNx7kDtp4mRWSLpH0uJn1R21/G91lHIjTX0i6wcyaJD0r6U8D11Px3P0hM/uupF+rOKL5MTG11iGZ2Y2SzpXUYWabJP2DpM9KutnMPqDij4W3l3SfTKkFAKglnIoEANQUgg0AUFMINgBATSHYAAA1hWADANQUgg0oEzObNLP+aY+SzfRhZt3TZ08H6hnXsQHl83t37wldBFDrOGIDAjOzDWb2f8zs4ehxfNS+1MzuMbM10fPRUftRZvY9M1sdPaamdEqa2Veje4X91MxaovU/amZPRtu5KdCfCZQNwQaUT8s+pyLfOe29YXc/U9KXVLxrgaLlb7n7ckk3SPrXqP1fJf3C3XMqzu/4RNR+gqRr3H2ZpB2S3ha1f0LS6dF2PhzPnwZUDmYeAcrEzHa5e/sB2jdIepO7PxtNiL3F3Rea2XZJi919T9Q+4O4dZjYoKePu49O20S3prujGjTKzj0tqdPf/bWZ3Stol6fuSvu/uu2L+U4GgOGIDKoMfZPlg6xzI+LTlSf2hD/2PJF0j6QxJj0Y3yARqFsEGVIZ3Tnt+MFp+QMU7F0jS/5B0X7R8j6SPSJKZJaO7YR+QmSUkZd39XhVvypqWtN9RI1BL+OUGlE/LtDs4SNKd7j415H+OmT2k4o/Nd0dtH5V0nZn9tYp3u56aff9KSauimdEnVQy5gYPsMynpO2aWkmSSvuDuO0r09wAViT42ILCoj63X3beHrgWoBZyKBADUFI7YAAA1hSM2AEBNIdgAADWFYAMA1BSCDQBQUwg2AEBN+f/EDKgPvPxY1QAAAABJRU5ErkJggg==\n"
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "#看模型的迭代情况\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "plt.plot(range(1, len(training_costs) + 1), training_costs)\n",
    "plt.tight_layout()\n",
    "plt.xlabel(\"Epochs\")\n",
    "plt.ylabel(\"Training Cost\")\n",
    "plt.show()"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "outputs": [],
   "source": [
    "#预测函数\n",
    "def predict_linreg(sess, model, X_test):\n",
    "    y_pred = sess.run(model.z_net,\n",
    "                      feed_dict={model.X: X_test})\n",
    "    return y_pred"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "outputs": [
    {
     "data": {
      "text/plain": "<Figure size 432x288 with 1 Axes>",
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAagAAAEYCAYAAAAJeGK1AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAmyUlEQVR4nO3de3RU1d3G8e8mhIRAFAiIFyQRsQRBrqnIwhcjqKWvlb4VRG2sYquACqJVbl64WQQUKQqCgghWorVagWpLUMpNilRJKVUEl1ikRigiCMgdkv3+sYPJwdwMM3PmzDyftVwy+0zm/Bq7fNx79vltY61FREQk2tTwuwAREZGyKKBERCQqKaBERCQqKaBERCQqKaBERCQq1fTz5g0bNrQZGRl+liAiIj7Lz8//ylrb6ORxXwMqIyODtWvX+lmCiIj4zBiztaxxLfGJiEhUUkCJiEhUUkCJiEhU8vU7qLIcO3aMgoICDh8+7HcpAiQnJ9OkSRMSExP9LkVE4kzUBVRBQQGpqalkZGRgjPG7nLhmrWXXrl0UFBRw3nnn+V2OiMSZqFviO3z4MGlpaQqnKGCMIS0tTbNZEfFF1AUUoHCKIvpnISLfkZsLGRlQo4b7e25uWG4TdUt8IiISxXJzoV8/OHjQvd661b0GyMkJ6a2icgZVFa1G5ZEx/M/l/tVqVF61PnfatGlkZ2dTu3ZtsrOzmT9//vf+jHvuuSek76tM3759adeuHVlZWcyaNavC9y5fvpzPPvssJPcVkTg0YkRJOJ1w8CA8+GDIbxXYGdSBI4WndL08AwcOZODAgTRv3pzly5dX6zOmTJkS0vdVxbRp02jZsiVt27alU6dOtGnTpsz3LV++nOzsbNRiSkS+tzffhM8/L/vaf/4T8tsFNqAiLTs7mx/+8If861//YvHixezfv5/evXtz4MABmjdvzpw5czzvPRFuo0eP5tixY7zzzjvs27ePvLw8zjzzzCq97/TTT+faa69l9+7dnH/++bRu3ZoHHnig3BrT0tK4+uqrWblyJQ0bNqRPnz4YY+jatSvjxo3j1ltvZdmyZSxYsIBWrVqRm5vLtm3bvvM+ERGPggK4+26oaEWpadOQ3zawS3yRtmbNGjp37szixYsB2L59O4MGDWLJkiV89tln7Nixo9yf3bx5MytXruTaa69l6dKlVX7fpk2baNKkCatWrWLz5s0VhtMJaWlp7Nmzhy+++IIJEyawaNEi3njjDQDmzJlD3759mTJlCrnFX2qW9T4REQCOH4ff/hZatqw4nFJSIAz/casZVBW1bt2aa6+99tvXiYmJPPfcc8yZM4fdu3dz6NChcn/25ptvBqBp06YcPXq0yu8755xzyM/Pp2vXrgwePLhKde7evZsmTZpQs2ZNxowZQ926dfnmm2/KfX9V3ycicea996B/f/jnP73jv/wl/PCHMGGCW9Zr2tSFU4g3SIACqsrq1q3reT179mx69+5Nnz59uOyyyyr82Tp16lTpHie/Ly8vj4cffpif/exnVfr5PXv2sGjRIu655x7GjRvHiBEjaNu2ref7qNq1a3Ow+AtOay2TJ08u830iEqf27oUHHoAZM8DakvGWLeGZZ6BrV/d6wICwl6Ilvmq68sorGT9+PN26dQPcUlmotW/fnkGDBtGtWzduuOEGPvzww3LfO2jQIHr06MHEiRPJzMzkJz/5CQMGDKBnz56kpKR8W1+vXr2YMGECl1xyCZ9++mm57xOROGMt/P73kJkJ06eXhFNyMjz6qJtJnQinCDG2dEJGWFZWlj35PKiNGzfSsmXLSn82Y/ifK33PZxOurnZt0WDWrFm8/PLLJCYmkpiYyP333092dnbE66jqPxMRCahPP4U774S33vKO9+gBTz8NzZqF9fbGmHxrbdbJ44Fd4quTlFDhVvI6SQkRrCY8br/9dm6//Xa/yxCRWHXkCDz+uPsOqXRLs7POgiefhN69oVQ3mVaj8ir99+6GMT1CVl5gAyqUvwQRkbizYoX7HmnTppIxY2DgQPjNb+C0077zI+F6/rQ8gQ0oERGphp07YcgQeOEF73iHDvDss5D1nZU232iThIhIPCgqgtmz3SaI0uGUmuqW8957L6rCCTSDEhGJfRs2uOW8Vau84716uXA65xx/6qpE8GdQIW77Pnr0aObNm1fmtcqau/bt25f27dvTuXNnrrvuOo4dO3ZKtWRnZzN06FAALrnkEkaPHl3pz/Tt27fSZrB+7AQUER8cPOiau7Zr5w2njAzXV++116I2nCDoAXWi7fvWrW7P/om272E6m6QqzV2nTp3Ku+++S926dVmyZMkp33P9+vUUFRVV+AyUiMh3LFoErVu7jg/Hj7uxmjVh+HA3o7o6+h/Die4lvuoclnfwINx0k/urItV4/quy5q4lH23Zv38/tWrVYseOHfTt25e9e/dyzTXXMGLECHbs2MF1113H4cOHadGiBZdddhm33XZbmfc8fvw4mzdvpmlxI8bdu3dzyy23sGfPHjp27MiUKVPYsmULOTk5pKSksG/fPoAy7ysiceCLL+Cee9zsqLQuXVwniNatfSmrOoI9g/JZWU1gBw0aREZGBo0bN6Zbt26MHz+e66+/ntWrV7NgwQJ27drF6tWr+dGPfsT8+fPZs2dPueEEcPbZZ7NkyRI6duwIwKOPPsoNN9zAO++8w969e8nLy+Oxxx5j6NCh5OXlfdtPr6z7ikgMKyyEp55yLYlKh1ODBvDcc7By5SmHU2XPl4b6+dPonkFFubKawE6dOpVVq1aRlJSEMYaPP/6Yd999l7lz53LgwAG2bdtGs2bNGDt2LIsWLar0e6X27dszd+5cbrzxRvbu3ctHH33EgOIeWJ06dWLjxo1s2bKFtm3bUrNmTdq1awdQ5n3T0tLC9rsQER+tXes2QeTne8dvucU9iNuoUUhuE+nnT6N7BmVtxX/Nm+favJeWkuLGK/vZECivCWz//v2ZPXs2hYWFtGjRggkTJrB8+XKGDx9OgwYNWLhwIc8//zyrVq3iiiuuqPAeHTp0YP369bQu/i+fVq1asWbNGsAdAdKqVSuaNm3Khg0bKCws5IMPPgAo874iEmP27XPnNHXq5A2nFi1g2TKYOzdk4eSH6A6oyuTkwMyZkJ7uvq9KT3evT7Ht+8iRI8nKyiIrK4tp06Z975+vX78+3bp1449//CPDhw9n0qRJdOnShby8PBo3bkzHjh3p1asX3bt35+abb66wQWuHDh1o06YNiYmJAIwYMYLf//73XHrppdSrV4+rrrqKoUOH8pvf/IYrr7ySWrVqAZR5XxGJEdbCq6+6Z5qmTnXPOAEkJcEjj8D69RADu3UD2yw2yEaPHs3f/vY3EhISqFmzJhMnTqRVq1Z+l1WuePhnIhIY//63a0e0aJF3/MorXRfy5s39qesUxFyz2CCryvNMIiIeR4/CE0/A2LHexq6NG8OUKXD99dXb+RzFojKgrLWYGPtFB5WfM2yRuJabCw8+6E6tPeMMSEiAbdtKrhsDd9zhOpHXq+dbmeEUdQGVnJzMrl27SEtLU0j5zFrLrl27SE5O9rsUkfhyoglB8enX7Njhvd6unXumqVOniJcWSVEXUE2aNKGgoICdO3f6XYrg/oOhSZMmfpchEl8eeKAknEozxi3zDRrkukLEuKj7X5iYmMh5553ndxkiIv7YuNEt65Xn3nsjV4vPgr3NXEQkVhw6BA89BG3blv+e4pZn8SLqZlAiInFn8WK48063hbw8KSluQ0Qc0QxKRMQv27fDDTdAjx7ecOrcGR59NORNCIJGMygRkUgrLHS78B54wLUrOqFePZg4EW67zZ1xF+enECigREQiad066N8f3n/fO37TTTBpknvwVgAFlIhIZHzzDYwc6Y7EONE7D+CCC2DGDOjeHYBWo/I4cKSw3I+pk5QQ8a7iflFAiYiEk7WwYIHrOl5QUDJeq5Zb4hs2DEo9DF9ROFXleiwJaUAZY+oDucAZQL61tn8oP19EJFC2bnWNXd980zvevbtr7PqDH/hTV0CEehffL4Dc4q60qcaY73SnFRGJeceOuYMCL7zQG05nnOHOq3v7bYVTFYR6iW8X0NoYUw84F/j85DcYY/oB/cCdRCsiElNWr3an2xYfHvqt/v1h/HioX9+fugIo1DOoVUA6cDewEdh98hustTOttVnW2qxGAT7pUUTEY/du1+C1SxdvOF10kQutZ55ROH1PoQ6oUcAAa+1YYBNwa4g/X0QkulgLL77oTredNatkPCXFLfPl57sHb+V7C3VA1QcuMsYkAJ0AHSYkIrHr44/hiivg5puh9AkMPXu6pq/33w+Jif7VF3ChDqjxwExgL9AAeDnEny8i4r/Dh2HUKGjTBpYuLRlv0gTmz4eFC6vd2LVOUsIpXY8lId0kYa19D2gVys8UEYkqS5a4xq6ffFIylpAAgwfDmDFQt+4pfXy8PIRbFXpQV0SkKnbsgPvuc6fdlnbxxfDss+6UWwkpdTMXEalIUZELoMxMbzidfrp72Hb1aoVTmGgGJSJSnvXr3TNNa9Z4x2+8ESZPhjPP9KeuOKEZlIjIyfbvdzvwOnb0htP557vDBV96SeEUAZpBiUjcqahj+BWf/J1HljzDWftKbRtPTIThw935TLVrR6hKUUCJSNwpK5zO2reTMUue5apPTlrOy852x2FkZkamOPmWAkpE4lbPDcsYtvJ3nL1vJ5aTvvNo2BCeeAJ+8Qt37LpEnAJKROJSzw3LeGzRUyQXHgOgdAS93OYqblz6EqSl+VOcANokISJx6LTD+3l08dPfhlNpX6bUY8SP71Y4RQEFlIjED2vh5Zf563MDqHvscJlvaXhwb4SLkvIooEQkPmzeDD/6Efz85zQ6sKfct207rWHkapIKKaBEJLYdOQKPPAKtW7uTbIvtSarDkQTv1/AHaybxWNebI12hlEMBJSKxa9kyaNsWRo50QQVQowYvdvopl94xhyE/HkzBaY0owlBwWiOG9xjIn1pdHlcdw6OZsda/I5uysrLs2rVrfbu/iMSoL790nSBefNE7npXl+up16OBPXVImY0y+tTbr5HHNoEQkdhQVuVNtMzO94ZSaClOnurZFCqfA0HNQIhIbPvzQNXb929+84336wG9/C2ef7U9dUm2aQYlIsB04AMOGQfv23nA67zxYtAheeUXhFFCaQYlIcL35JgwcCFu3lowlJsKQIfDgg5CS4l9tcsoUUCISPAUF7oj111/3jnft6hq7XnihP3VJSGmJT0SC4/hxmDIFWrb0hlNaGjz/PCxfrnCKIZpBiUgwvPee2wSxbp13/NZb4bHHXPdxiSmaQYlIdNu7133PdMkl3nBq2RJWrHAzJ4VTTFJAiUh0stbtwMvMhKefdq8BkpPh0Ufhn/903zlJzNISn4iERUXHqgPUSUpgw5geZV/89FO46y5YvNg73qOHC6tmzUJYqUQrzaBEJCwqCqdyrx89CuPGucaupcPprLPgD3+Av/xF4RRHNIMSkeiwYgXccQds3FgyZoz7/umRR+D00/2rTXyhgBIRf331lWvs+sIL3vEOHVxj16zv9BCVOKElPhHxhbFFbgdeixbecEpNhSefdNvKFU5xTTMoEYm4C3Zu5TdvTYeCDd4LvXq5cDrnHH8Kk6iigBKRiEk+dphBq1+h33uvk1hUapNERgZMmwZXX+1bbRJ9FFAiAXZKW7kjpOeGZQxd+TvO2beTQlODmrao5GLNmu77p4cfVmNX+Q4FlEiAVWsrd4TUSUqg+z+WMDFvKrWPHwXwhNM/zr2QDn95xW0pFymDAkpEwmLDyCuh0c+hOJw8GjSgw2cfQA3t05Ly6f8dIhJ6+fnQqRN8/XXZ17/+WuEkldL/Q0QkdPbtc+c0XXyxC6nyNG0auZoksLTEJyKnzlp47TUXTtu3l4wnJLiZ0rFjJWMpKa6dkUglNIMSkVOzZYvbHt6njzecrroKNm2COXMgPd21LUpPh5kzISfHv3olMDSDEpHqOXoUJk+GsWPh0KGS8caN3am311/vQql5cwWSVIsCSiTA6iQlVPocVFisWuVOt91QqhOEMa7Z67hxUK9eeO4rcSVsAWWMmQ4ssta+Ea57iMS7iD+Eu2sXDBsGs2d7x9u1g2eecTv3REIkLN9BGWP+BzhT4SQSI6x1DV0zM73hVKeOW+Z7/32Fk4RcyAPKGJMIzAI+M8b8tIzr/Ywxa40xa3fu3Bnq24tIqG3cCJdfDn37uqMxTvjZz9y1e+91LYtEQiwcM6ibgY+Ax4CLjTGDSl+01s601mZZa7MaNWoUhtuLSEgcOgQPPQRt27rDBE9o2hT+9Cd4/XU491z/6pOYF46Aag/MtNb+F5gHXB6Ge4hIOL31Flx0kdvwcOIZpoQEGDIEPvoIrrnG3/okLoRjXr4ZaFb85yxgaxjuISLhsH27W7J75RXveOfObhNEmzb+1CVxKRwBNRt43hhzA5AI9A7DPUQklAoL3fHqI0a4dkUn1KsHEyfCbbepd55EXMgDylr7DXBdqD9XRMJk3Tro39/txCvtpptg0iT34K2ID/SfRCLx6ptv3HJeVpY3nC64AJYsgRdfVDiJr7Q3VCTeWAsLFsDdd0NBQcl4rVrwwAPuQdzkZN/KEzlBASUST7ZuhYED4c03vePdu8P06fCDH/hTl0gZtMQnEg+OHYPHH4cLL/SG0xlnwLx58PbbCieJOppBicS61atdY9cPPvCO9+8P48dD/fr+1CVSCQWUSKzavRuGD4dZs7zjF13ktpR37uxPXSJVpCU+kVhjrduBl5npDaeUFLfMl5+vcJJA0AxKJBbk5sKDD8J//uN24x054r1+zTUwdao70VYkIBRQIkGXmwv9+sHBg+516XBq0sQF009/6g4UFAkQBZRI0P361yXhVFpqqmvsmpoa+ZpEQkDfQYkE1Y4dkJMDX35Z9vX9+xVOEmgKKJGgKSpyncVbtICXXir/fU2bRq4mkTDQEp9IkKxf755pWrPGO56Q4DqSn5CS4s5yEgkwzaBEgmD/frj/fujY0RtO558PixfDCy+4HXrGuL/PnOmW/0QCTDMokUq0GpXHgSOF5V6vk5TAhjE9wlfAwoUwaBB8/nnJWGKiewh3xAioXduNKZAkxiigRCpRUThV5Xq1/ec/ruP4woXe8exsmDHDPYgrEsO0xCcSbY4fhyeecI1dS4dTw4ZuKW/pUoWTxAXNoESiyZo1bhPE+vXe8dtugwkTIC3Nn7pEfKCAEokGX3/tDgt89lnXS++EVq3clvJLL/WvNhGfaIlPxE/WumeZMjNdEJ0Ip9q13Yxp3TqFk8QtzaBE/PLJJ3DnnbBkiXf8f/8Xpk2D887zpy6RKFGlGZQxpo8xJincxYjEhSNHYOxYdy5T6XA6+2x47TV34q3CSaTKS3wtgWXGmGeNMV3CWZBItKmTlHBK1z2WLoU2bWDUqJKu4zVqwODBsHEj9OqlruMixYwt/YVsZW825jLgeaAIGGetnXsqN8/KyrJr1649lY8QCYYvv4T77oN587zjWVluY0SHDv7UJRIFjDH51tqsk8e/zxLfAmAkMBG4BLgjpBWKxKKiIneqbWamN5xSU905TWvWKJxEylHVTRIXAr+21v77xIAx5tbwlCQSIz74wD3TtHq1d7xPH/jtb913TiJSrioFlLV2dBljH4W8GpFYcOCA2wQxebLrCnHCeefB9OnQI4x9+0RiiLaZi4TSm2/CwIGwdWvJWGIiDBkCDz7ojsEQkSpRQImEQkGBa+w6f753vGtX19j1wgv9qUskwNRJQuRUHD8OU6ZAy5becEpLg+efh+XLFU4i1aQZlEh1vfee2wSxbp13/NZb4bHHXPdxEak2zaBEqio3FzIy3IO1qanQqZM3nFq2hBUr3MxJ4SRyyjSDEqmK3Fzo1w8OHnSv9+8vuZacDCNHugdxa9Xypz6RGKSAEqmKoUNLwqm05GTYsAGaNYt8TSIxTgElUpEjR+Dxx2HbtvKvK5xEwkIBJVKeFSvcJohNm8p/T9OmkatHJM5ok4TIyXbuhL59ITvbG04ndxlPSYFx4yJZmUhcUUCJnFBUBLNnu8auL7xQMp6aCk8+6cbS011QpafDzJmQk+NfvSIxTkt8IuA2OgwYAKtWecd79XLhdM457vUvfhH52kTiVFhmUMaYxsaYdZW/U8RnBw/CiBHQrp03nDIyXF+9114rCScRiahwzaAmAbXD9NkiobFoEdx1F2zZUjJWs6Z7nmnkSDV2FfFZyAPKGNMNOAD8t5zr/YB+AE21A0r8sG2bO2L9tde84126wDPPQOvW/tQlIh4hXeIzxtQCHgaGl/cea+1Ma22WtTarUaNGoby9SMUKC90ptpmZ3nBq0ACeew5WrlQ4iUSRUM+ghgPTrbV7zMlbckX8lJ8P/fu7v5d2yy3uQVz9x5JI1An1JokrgLuMMcuBdsaY50L8+SLfz759bjnv4ou94dSiBSxbBnPnKpxEolRIZ1DW2q4n/myMWW6tvS2Uny9SZda6ZbzBg2H79pLxpCR46CF3wm1Skn/1iUilwvYclLU2O1yfLVKhLVvc7rxFi7zjV14J06dD8+b+1CUi34s6SUjsOHoUJkyAVq284dS4Mbz8MixerHASCRB1kpDYsGqV6wSxYUPJmDFwxx2uX169er6VJiLVo4CSYNu1C4YNcz30SmvXzj3T1KmTL2WJyKnTEp8Ek7WueWtmpjec6tSByZPh/fcVTiIBpxmUBM+mTW45b8UK7/j//R889RSce64vZYlIaGkGJcFx6BA8/DC0aeMNp6ZNYeFCmD9f4SQSQzSDkmB46y2480749NOSsYQE+PWvYdQot7QnIjFFASXRbft2uPdeeOUV73jnzm4TRJs2/tQlImGnJT6JLrm57iymGjVcE9dmzbzhVK8ePPus21aucBKJaZpBSfTIzYV+/dwhggBff+29ftNNMGmSe/BWRGKeAkqix4gRJeFUWs2akJcH3btHviYR8Y0CSvxnLSxYAJ9/Xvb1wkKFk0gcUkDFkVaj8jhwpLDc63WSEtgwpkcEKwK2boVBg+CNN8p/j05eFolL2iQRRyoKp6pcD6ljx9xBgRdeWHE4paS4XnoiEncUUBJ5q1dDx44wdKj3O6f+/d0OvfR01+g1PR1mzoScHP9qFRHfaIlPImf3brcRYuZM7/hFF7lg6tzZve7XL/K1iUjU0QxKws9amDfPNXYtHU4pKW6ZLz+/JJxERIppBiXh9fHHrkXR0qXe8Z49YepUbYAQkXJpBiXhcfiw65HXpo03nJo0cU1dFy5UOIlIhTSDktBbssTNmj75pGQsIQEGD4YxY6BuXf9qE5HAUEDFkTpJCZU+B3VKduyA++5zLYtKu/hitwmiXbtT+3wRiSsKqDgStodwi4pg1iwYPhz27CkZP/10GD/e7cpLOMXwE5G4o4CSU/Ovf7nnl9as8Y7feKM7ev3MM/2pS0QCT5skpHr274chQ6BDB284nX8+LF4ML72kcBKRU6IZlHx/f/oTDBzobe6amOiW+EaMgNq1/atNRGKGAkqq7vPPXWPXhQu949nZMGOGexBXRCREtMQnlTt+3H2f1LKlN5waNoTf/c4956RwEpEQ0wxKKvb3v7tNEOvXe8dvuw0mTnTHsouIhIFmUFK2PXvcw7adO3vDqXVrWLXKbStXOIlIGCmgxMtaePllt2Q3Y4Z7DW7jw8SJ8I9/QJcu/tYoInFBS3xSYvNmN2t6+23v+NVXw7RpkJHhS1kiEp80gxI4cgQeecQt35UOp3POgT/+0Z14q3ASkQjTDCreLVsGd9zhjsU4oUYNuPtuGDsWUlP9q01E4poCKh7l5rqHagsKvnstK8s1du3QIfJ1iYiUooCKN/Pmwa9+BUePeseTk2HSJBgwQI1dRSQqKKDiyYcfuueXTg4ngLQ0uOuuyNckIlIObZKIBwcOwLBh0L692xBRlm3bIluTiEglNIOKdX/+s5sZbd1a8ft8Pn691ai8Sg9TDNt5ViISlTSDilUFBdCrF/zkJ95watHCfd9UWkoKjBsX2fpOUlE4VeW6iMSekAeUMeZ0Y8wiY8xbxpj5xphaob6HVOD4cXjySdfY9fXXS8bT0mDOHNi4EZ57DtLTwRj395kzISfHv5pFRMoQjiW+HGCytfZtY8wMoAfwpzDcR072/vuuseu6dd7xX/7StSlq2NC9zslRIIlI1At5QFlrp5d62Qj4svR1Y0w/oB9AU5+/94gZe/fCgw/C9OklvfPAzaKeeQa6dvWvNhGRagrbd1DGmM5AfWvtmtLj1tqZ1tosa21Wo0aNwnX7+GAt/OEPLoiefroknJKT4dFH4Z//VDiJSGCFZRefMaYBMBXoFY7PF+DTT92x63l53vEf/9g1dm3WzJ+6RERCJBybJGoBrwIjrLWV7G2W7+3oUbfjrnVrbziddZabTf35zwonEYkJ4Vji+xXQAXjQGLPcGHN9GO4Rn1asgHbt4KGH4PBhN2YMDBoEmzbBdde51wFUJ6ni9kqVXReR2GNs6S/VIywrK8uuXbvWt/sHxldfwZAhMHeud7xDB9fYNSvLl7JERELBGJNvrf3Ov8j0oG40KyqC5593D9eWDqfUVPes03vvKZxEJGap1VG02rDBndP0zjve8d69YcoUd5igiEgM0wwq2hw8CA884L5rKh1OGRluA8SrryqcRCQuaAYVTRYtco1dt2wpGatZE+6/Hx5+2PXMExGJEwqoaLBtG9xzj5sdldali+sE0bq1L2WJiPhJS3x+KiyEqVMhM9MbTg0auIauK1cqnEQkbmkG5Zf8fNfYNT/fO37LLfD446A2UCIS5xRQIVbZwXtncJj3Dq107YiKikoutGjhlvOys8NfpIhIACigQqzccLKWH3/8N0b9dSbs310ynpTkOkMMGeL+LCIigAIqIprs+S+PvD2Dy/990nLeVVe5LuTNm/tTmIhIFFNAhUnPDcsYtvJ3nL1vJ5aTdqM0buwetr3++sD2zhMRCTcFVBj03LCMxxY9RXLhMQBORFARMK/91dy8dB7Uq+dXeSIigaBt5iFW79A+xi9++ttwKu2rOvUZedUdCicRkSpQQIWKtfDCC/x11gDqHDtc5lsaHtgT2ZpERAIskEt8lW3lrpOUwIYxPSJX0MaNrrHrihWkVfC2bac1jFhJIiJBF8gZVEXhVJXrIXPokNsi3ratO0yw2K7kVI4kJHreerBmEo91vVkH74mIVFEgAyoqvPUWXHSRO379WPH3TQkJMGQIaV9tJ+mFOZCe7nbppaeTMnc2T70xKbIzOxGRAAvkEp+vtm+He++FV17xjnfu7DpBtGnjXufkuL9ERKRaNIOqqsJCmD7dNXYtHU716rlj11etKgknERE5ZZpBVcW6dTBggDtivbSbboInnoAzzvCnLhGRGKYZVEW++cYt52VlecPpggtgyRJ48UWFk4hImGgGVRZrYcECuPtuKCgoGa9Vyx3HPmwYJCf7Vp6ISDwIZEDVSUqo9Dmoatu6FQYOhDff9I537+6+g/rBD6r/2SIiUmWBDKiwbNU+dsw1cB09Gg4eLBk/4wyYPBl+/nM1dhURiaBABlTIrV7tNkF88IF3vH9/GD8e6tf3py4RkTgW3wG1ezeMGAEzZ3rH27RxzzR17uxPXSIiEqe7+Kx1O/AyM73hlJICjz8Oa9cqnEREfBZ/M6iPP4Y774SlS73jPXvC1KnQtKk/dYmIiEf8zKAOH4ZRo9zyXelwatIE5s+HhQsVTiIiUSQ+ZlBLlrhZ0yeflIwlJMDgwTBmDNSt619tIiJSptgOqB074L77IDfXO37xxa5/Xrt2vpQlIiKVi80lvqIiF0CZmd5wOv1097Dt6tUKJxGRKBd7M6j1690zTWvWeMdvvNE9cHvmmf7UJSIi30vszKD274f774eOHb3hdP75sHgxvPSSwklEJECCHVC5uZCRATVquHOZnnjCndsEkJgIDz/sukNcdZWfVYqISDUEd4kvNxduvx0OHXKvC0s1j83Ohhkz3HdQIiISSMENqGHDSsKptLQ095yTGruKiARacJf4tm0re3z3boWTiEgMCG5Aldf1Qd0gRERiQlgCyhgz2xjzrjHmoXB8PgDjxrnmrqWlpLhxEREJvJAHlDHmWiDBWtsZaGaMuSDU9wAgJ8d1Ik9Pd0t66enudU5OWG4nIiKRFY5NEtnAH4r//BZwKfBJue8+FTk5CiQRkRgVjiW+OsAXxX/eDTQufdEY088Ys9YYs3bnzp1huL2IiMSCcATUfqB28Z/rnnwPa+1Ma22WtTarUaNGYbi9iIjEgnAEVD5uWQ+gLfBZGO4hIiIxLhzfQS0A3jHGnA38GLgkDPcQEZEYF/IZlLV2H26jxBrgcmvt3lDfQ0REYl9YWh1Za7+mZCefiIjI92astf7d3JidwNYQfFRD4KsQfE680e+tevR7qx793qonHn5v6dba7+ya8zWgQsUYs9Zam+V3HUGj31v16PdWPfq9VU88/96C24tPRERimgJKRESiUqwE1Ey/Cwgo/d6qR7+36tHvrXri9vcWE99BiYhI7ImVGZSIiMQYBZSIiESlwAdURA5HjDHGmNONMYuMMW8ZY+YbY2r5XVOQGGMaG2PW+V1H0BhjphtjrvG7jqAwxtQ3xvyl+PSHZ/2uxw+BDqiIHY4Ye3KAydbaq4D/Aj18ridoJlHSsV+qwBjzP8CZ1to3/K4lQH4B5BY/A5VqjIm7Z6ECHVCUfTiiVMJaO91a+3bxy0bAl37WEyTGmG7AAVywSxUYYxKBWcBnxpif+l1PgOwCWhtj6gHnAp/7W07kBT2gKjwcUSpmjOkM1LfWrvG7liAoXgp9GBjudy0BczPwEfAYcLExZpDP9QTFKiAduBvYiPt3XFwJekBVeDiilM8Y0wCYCvzS71oCZDgw3Vq7x+9CAqY9MNNa+19gHnC5z/UExShggLV2LLAJuNXneiIu6P9C1+GI1VA8E3gVGGGtDUWz3nhxBXCXMWY50M4Y85zP9QTFZqBZ8Z+zCE2D6HhQH7jIGJMAdALi7qHVQD+oa4w5DXgH+CvFhyPq/KnKGWPuAB4F1hcPzbDWvuJjSYFjjFlurc32u44gMMakAs/jluATgd7W2i8q/ikxxlwMzMEt870L/Mxau9/fqiIr0AEFbismcCWwsngJQUREYkDgA0pERGJT0L+DEhGRGKWAEhGRqKSAEhGRqKSAEhGRqKSAEhGRqKSAEvGBMeZSY8yrxpgaxd34z/W7JpFoo23mIj4xxswBDgFbrbUT/a5HJNoooER8Unx8wrtAI/X3E/kuBZSIT4wxC4B/AXWstff5XI5I1NF3UCI+MMb0BrZZa0cCrYwxHfyuSSTaaAYlIiJRSTMoERGJSgooERGJSgooERGJSgooERGJSgooERGJSgooERGJSv8PmrqTHaUNbZ0AAAAASUVORK5CYII=\n"
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.scatter(X_train, y_train, marker='s', s=50,\n",
    "            label=\"Training Data\")\n",
    "plt.plot(X_train, predict_linreg(sess, lrmodel, X_train),\n",
    "         color='red', marker='o',\n",
    "         markersize=6, linewidth=3,\n",
    "         label=\"LinReg Model\")\n",
    "plt.xlabel(\"x\")\n",
    "plt.ylabel(\"y\")\n",
    "plt.legend()\n",
    "plt.tight_layout()\n",
    "plt.show()"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "## 使用tensorflow的高级API高效率的训练神经网络"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "outputs": [],
   "source": [
    "#训练手写识别数字数据\n",
    "minist = np.load(\"data/mnist_scaled.npz\")\n",
    "X_train_centered, y_train, X_test_centered, y_test = [minist[i] for i in minist]"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/b6/sj0f93ps3rn4fdfsd4mkgj_00000gn/T/ipykernel_1960/3033739132.py:15: UserWarning: `tf.layers.dense` is deprecated and will be removed in a future version. Please use `tf.keras.layers.Dense` instead.\n",
      "  h1=tf.layers.dense(inputs=tf_x,units=50,\n",
      "/Users/apple/opt/anaconda3/envs/Deep-Learning/lib/python3.10/site-packages/keras/legacy_tf_layers/core.py:261: UserWarning: `layer.apply` is deprecated and will be removed in a future version. Please use `layer.__call__` method instead.\n",
      "  return layer.apply(inputs)\n",
      "/var/folders/b6/sj0f93ps3rn4fdfsd4mkgj_00000gn/T/ipykernel_1960/3033739132.py:18: UserWarning: `tf.layers.dense` is deprecated and will be removed in a future version. Please use `tf.keras.layers.Dense` instead.\n",
      "  h2=tf.layers.dense(inputs=h1,units=50,\n",
      "/var/folders/b6/sj0f93ps3rn4fdfsd4mkgj_00000gn/T/ipykernel_1960/3033739132.py:21: UserWarning: `tf.layers.dense` is deprecated and will be removed in a future version. Please use `tf.keras.layers.Dense` instead.\n",
      "  logits=tf.layers.dense(inputs=h2,units=10,\n"
     ]
    }
   ],
   "source": [
    "n_features = X_train_centered.shape[1]\n",
    "n_classes = 10\n",
    "random_seed = 123\n",
    "np.random.seed(random_seed)\n",
    "g = tf.Graph()  #创建计算图\n",
    "with g.as_default():\n",
    "    tf.set_random_seed = random_seed\n",
    "    tf_x = tf.placeholder(dtype=tf.float32,  #输入x\n",
    "                          shape=(None, n_features),\n",
    "                          name=\"tf_x\")\n",
    "    tf_y = tf.placeholder(dtype=tf.int32,  #输入y\n",
    "                          shape=None,\n",
    "                          name=\"tf_y\")\n",
    "    y_onehot = tf.one_hot(indices=tf_y, depth=n_classes)  #将y转换编码，多分类模型\n",
    "    h1 = tf.layers.dense(inputs=tf_x, units=50,  #输入层\n",
    "                         activation=tf.tanh,\n",
    "                         name='layer1')\n",
    "    h2 = tf.layers.dense(inputs=h1, units=50,  #隐藏层\n",
    "                         activation=tf.tanh,\n",
    "                         name='layer2')\n",
    "    logits = tf.layers.dense(inputs=h2, units=10,  #输出层\n",
    "                             activation=None,\n",
    "                             name='layer3')\n",
    "    predictions = {\n",
    "        'classes': tf.argmax(logits, axis=1,\n",
    "                             name='predicted_classes'),\n",
    "        'probabilities': tf.nn.softmax(logits,\n",
    "                                       name='softmax_tensor')\n",
    "    }"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--Epoch  1 Avg.Training Loss: 1.8044\n",
      "--Epoch  2 Avg.Training Loss: 1.1731\n",
      "--Epoch  3 Avg.Training Loss: 0.8922\n",
      "--Epoch  4 Avg.Training Loss: 0.7282\n",
      "--Epoch  5 Avg.Training Loss: 0.6273\n",
      "--Epoch  6 Avg.Training Loss: 0.5601\n",
      "--Epoch  7 Avg.Training Loss: 0.5120\n",
      "--Epoch  8 Avg.Training Loss: 0.4760\n",
      "--Epoch  9 Avg.Training Loss: 0.4478\n",
      "--Epoch 10 Avg.Training Loss: 0.4252\n",
      "--Epoch 11 Avg.Training Loss: 0.4065\n",
      "--Epoch 12 Avg.Training Loss: 0.3908\n",
      "--Epoch 13 Avg.Training Loss: 0.3773\n",
      "--Epoch 14 Avg.Training Loss: 0.3655\n",
      "--Epoch 15 Avg.Training Loss: 0.3552\n",
      "--Epoch 16 Avg.Training Loss: 0.3459\n",
      "--Epoch 17 Avg.Training Loss: 0.3376\n",
      "--Epoch 18 Avg.Training Loss: 0.3301\n",
      "--Epoch 19 Avg.Training Loss: 0.3232\n",
      "--Epoch 20 Avg.Training Loss: 0.3168\n",
      "--Epoch 21 Avg.Training Loss: 0.3109\n",
      "--Epoch 22 Avg.Training Loss: 0.3055\n",
      "--Epoch 23 Avg.Training Loss: 0.3004\n",
      "--Epoch 24 Avg.Training Loss: 0.2956\n",
      "--Epoch 25 Avg.Training Loss: 0.2911\n",
      "--Epoch 26 Avg.Training Loss: 0.2868\n",
      "--Epoch 27 Avg.Training Loss: 0.2827\n",
      "--Epoch 28 Avg.Training Loss: 0.2789\n",
      "--Epoch 29 Avg.Training Loss: 0.2752\n",
      "--Epoch 30 Avg.Training Loss: 0.2717\n",
      "--Epoch 31 Avg.Training Loss: 0.2683\n",
      "--Epoch 32 Avg.Training Loss: 0.2651\n",
      "--Epoch 33 Avg.Training Loss: 0.2620\n",
      "--Epoch 34 Avg.Training Loss: 0.2590\n",
      "--Epoch 35 Avg.Training Loss: 0.2561\n",
      "--Epoch 36 Avg.Training Loss: 0.2533\n",
      "--Epoch 37 Avg.Training Loss: 0.2506\n",
      "--Epoch 38 Avg.Training Loss: 0.2479\n",
      "--Epoch 39 Avg.Training Loss: 0.2454\n",
      "--Epoch 40 Avg.Training Loss: 0.2429\n",
      "--Epoch 41 Avg.Training Loss: 0.2405\n",
      "--Epoch 42 Avg.Training Loss: 0.2382\n",
      "--Epoch 43 Avg.Training Loss: 0.2359\n",
      "--Epoch 44 Avg.Training Loss: 0.2337\n",
      "--Epoch 45 Avg.Training Loss: 0.2315\n",
      "--Epoch 46 Avg.Training Loss: 0.2294\n",
      "--Epoch 47 Avg.Training Loss: 0.2274\n",
      "--Epoch 48 Avg.Training Loss: 0.2254\n",
      "--Epoch 49 Avg.Training Loss: 0.2234\n",
      "--Epoch 50 Avg.Training Loss: 0.2215\n"
     ]
    }
   ],
   "source": [
    "#定义成本函数\n",
    "with g.as_default():\n",
    "    cost = tf.losses.softmax_cross_entropy(\n",
    "        onehot_labels=y_onehot, logits=logits\n",
    "    )\n",
    "    optimizer = tf.train.GradientDescentOptimizer(\n",
    "        learning_rate=0.001\n",
    "    )\n",
    "    train_op = optimizer.minimize(loss=cost)\n",
    "    init_op = tf.global_variables_initializer()\n",
    "\n",
    "\n",
    "#定义一个函数生成大批量数据\n",
    "def creat_batch_generator(X, y, batch_size=128, shuffle=False):\n",
    "    X_copy = np.array(X)\n",
    "    y_copy = np.array(y)\n",
    "    if shuffle:\n",
    "        data = np.column_stack((X_copy, y_copy))\n",
    "        np.random.shuffle(data)\n",
    "        X_copy = data[:, :-1]\n",
    "        y_copy = data[:, -1].astype(int)\n",
    "    for i in range(0, X.shape[0], batch_size):\n",
    "        yield (X_copy[i:i + batch_size, :], y_copy[i:i + batch_size])\n",
    "\n",
    "\n",
    "##初始化tensorflow的变量并开始训练\n",
    "sess = tf.Session(graph=g)\n",
    "\n",
    "sess.run(init_op)\n",
    "\n",
    "##迭代50次,每一次迭代都是将原始数据分成64小份的数据集进行计算，每一次迭代算出所有小份数据集的平均损失\n",
    "for epoch in range(50):\n",
    "    training_costs = []\n",
    "    batch_generator = creat_batch_generator(  #该函数返回的是一个生成器\n",
    "        X_train_centered, y_train, batch_size=64\n",
    "    )\n",
    "    for batch_X, batch_y in batch_generator:  #每次调用生成器，就是产生64大小的小份数据集，用该小份数据集训练模型\n",
    "        feed = {tf_x: batch_X, tf_y: batch_y}\n",
    "        _, batch_cost = sess.run([train_op, cost], feed_dict=feed)\n",
    "        training_costs.append(batch_cost)\n",
    "    print('--Epoch %2d Avg.Training Loss: %.4f' % (epoch + 1, np.mean(training_costs)))"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test Accuracy:93%\n"
     ]
    }
   ],
   "source": [
    "feed = {tf_x: X_test_centered, tf_y: y_test}\n",
    "y_pred = sess.run(predictions['classes'], feed_dict=feed)\n",
    "print(\"Test Accuracy:%2.f%%\" % (100 * np.sum(y_pred == y_test) / y_test.shape[0]))"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "## 用keras研发多层神经网络"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "outputs": [],
   "source": [
    "minist = np.load(\"data/mnist_scaled.npz\")\n",
    "X_train_centered, y_train, X_test_centered, y_test = [minist[i] for i in minist]\n",
    "n_features = X_train_centered.shape[1]\n",
    "n_classes = 10\n",
    "random_seed = 123\n",
    "np.random.seed(random_seed)\n",
    "import tensorflow.keras as keras"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/apple/opt/anaconda3/envs/Deep-Learning/lib/python3.10/site-packages/keras/optimizer_v2/gradient_descent.py:102: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
      "  super(SGD, self).__init__(name, **kwargs)\n"
     ]
    }
   ],
   "source": [
    "#用keras自带的onehot转换编码\n",
    "tf.set_random_seed = random_seed\n",
    "y_train_onehot = keras.utils.to_categorical(y_train)\n",
    "model = keras.models.Sequential()\n",
    "model.add(\n",
    "    keras.layers.Dense(\n",
    "        units=50,\n",
    "        input_dim=X_train_centered.shape[1],\n",
    "        kernel_initializer='zeros',\n",
    "        activation='tanh'\n",
    "    )\n",
    ")\n",
    "model.add(\n",
    "    keras.layers.Dense(\n",
    "        units=50,\n",
    "        input_dim=50,\n",
    "        kernel_initializer='glorot_uniform',  #权重矩阵的初始化算法\n",
    "        bias_initializer='zeros',\n",
    "        activation='tanh'\n",
    "    )\n",
    ")\n",
    "model.add(\n",
    "    keras.layers.Dense(\n",
    "        units=y_train_onehot.shape[1],\n",
    "        input_dim=50,\n",
    "        kernel_initializer='glorot_uniform',\n",
    "        bias_initializer='zeros',\n",
    "        activation='softmax'\n",
    "    )\n",
    ")\n",
    "sgd_optimizer = keras.optimizers.SGD(\n",
    "    lr=0.001, decay=1e-7, momentum=0.9\n",
    ")\n",
    "model.compile(optimizer=sgd_optimizer,\n",
    "              loss='categorical_crossentropy')  #二分类变量的损失熵作为损失函数"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/50\n",
      "844/844 [==============================] - 5s 4ms/step - loss: 0.6479 - val_loss: 0.3371\n",
      "Epoch 2/50\n",
      "844/844 [==============================] - 3s 4ms/step - loss: 0.3538 - val_loss: 0.2606\n",
      "Epoch 3/50\n",
      "844/844 [==============================] - 3s 4ms/step - loss: 0.2964 - val_loss: 0.2307\n",
      "Epoch 4/50\n",
      "844/844 [==============================] - 3s 4ms/step - loss: 0.2633 - val_loss: 0.2108\n",
      "Epoch 5/50\n",
      "844/844 [==============================] - 3s 4ms/step - loss: 0.2392 - val_loss: 0.1901\n",
      "Epoch 6/50\n",
      "844/844 [==============================] - 3s 4ms/step - loss: 0.2210 - val_loss: 0.1788\n",
      "Epoch 7/50\n",
      "844/844 [==============================] - 3s 4ms/step - loss: 0.2056 - val_loss: 0.1684\n",
      "Epoch 8/50\n",
      "844/844 [==============================] - 3s 4ms/step - loss: 0.1933 - val_loss: 0.1591\n",
      "Epoch 9/50\n",
      "844/844 [==============================] - 3s 4ms/step - loss: 0.1824 - val_loss: 0.1508\n",
      "Epoch 10/50\n",
      "844/844 [==============================] - 3s 4ms/step - loss: 0.1726 - val_loss: 0.1450\n",
      "Epoch 11/50\n",
      "844/844 [==============================] - 3s 4ms/step - loss: 0.1634 - val_loss: 0.1422\n",
      "Epoch 12/50\n",
      "844/844 [==============================] - 3s 4ms/step - loss: 0.1564 - val_loss: 0.1362\n",
      "Epoch 13/50\n",
      "844/844 [==============================] - 3s 4ms/step - loss: 0.1490 - val_loss: 0.1352\n",
      "Epoch 14/50\n",
      "844/844 [==============================] - 3s 4ms/step - loss: 0.1430 - val_loss: 0.1275\n",
      "Epoch 15/50\n",
      "844/844 [==============================] - 3s 4ms/step - loss: 0.1369 - val_loss: 0.1256\n",
      "Epoch 16/50\n",
      "844/844 [==============================] - 3s 4ms/step - loss: 0.1309 - val_loss: 0.1207\n",
      "Epoch 17/50\n",
      "844/844 [==============================] - 3s 4ms/step - loss: 0.1263 - val_loss: 0.1197\n",
      "Epoch 18/50\n",
      "844/844 [==============================] - 3s 4ms/step - loss: 0.1222 - val_loss: 0.1171\n",
      "Epoch 19/50\n",
      "844/844 [==============================] - 3s 4ms/step - loss: 0.1173 - val_loss: 0.1149\n",
      "Epoch 20/50\n",
      "844/844 [==============================] - 3s 4ms/step - loss: 0.1132 - val_loss: 0.1151\n",
      "Epoch 21/50\n",
      "844/844 [==============================] - 3s 4ms/step - loss: 0.1093 - val_loss: 0.1096\n",
      "Epoch 22/50\n",
      "844/844 [==============================] - 4s 4ms/step - loss: 0.1061 - val_loss: 0.1064\n",
      "Epoch 23/50\n",
      "844/844 [==============================] - 4s 4ms/step - loss: 0.1027 - val_loss: 0.1114\n",
      "Epoch 24/50\n",
      "844/844 [==============================] - 3s 4ms/step - loss: 0.0993 - val_loss: 0.1063\n",
      "Epoch 25/50\n",
      "844/844 [==============================] - 3s 4ms/step - loss: 0.0965 - val_loss: 0.1034\n",
      "Epoch 26/50\n",
      "844/844 [==============================] - 4s 5ms/step - loss: 0.0932 - val_loss: 0.1048\n",
      "Epoch 27/50\n",
      "844/844 [==============================] - 3s 4ms/step - loss: 0.0904 - val_loss: 0.0990\n",
      "Epoch 28/50\n",
      "844/844 [==============================] - 4s 4ms/step - loss: 0.0877 - val_loss: 0.1044\n",
      "Epoch 29/50\n",
      "844/844 [==============================] - 4s 4ms/step - loss: 0.0855 - val_loss: 0.0952\n",
      "Epoch 30/50\n",
      "844/844 [==============================] - 3s 4ms/step - loss: 0.0827 - val_loss: 0.0993\n",
      "Epoch 31/50\n",
      "844/844 [==============================] - 4s 5ms/step - loss: 0.0806 - val_loss: 0.0937\n",
      "Epoch 32/50\n",
      "844/844 [==============================] - 4s 5ms/step - loss: 0.0781 - val_loss: 0.0957\n",
      "Epoch 33/50\n",
      "844/844 [==============================] - 4s 4ms/step - loss: 0.0765 - val_loss: 0.0919\n",
      "Epoch 34/50\n",
      "844/844 [==============================] - 3s 4ms/step - loss: 0.0742 - val_loss: 0.0901\n",
      "Epoch 35/50\n",
      "844/844 [==============================] - 3s 4ms/step - loss: 0.0726 - val_loss: 0.0904\n",
      "Epoch 36/50\n",
      "844/844 [==============================] - 4s 4ms/step - loss: 0.0705 - val_loss: 0.0905\n",
      "Epoch 37/50\n",
      "844/844 [==============================] - 3s 4ms/step - loss: 0.0686 - val_loss: 0.0878\n",
      "Epoch 38/50\n",
      "844/844 [==============================] - 3s 4ms/step - loss: 0.0671 - val_loss: 0.0909\n",
      "Epoch 39/50\n",
      "844/844 [==============================] - 3s 4ms/step - loss: 0.0654 - val_loss: 0.0900\n",
      "Epoch 40/50\n",
      "844/844 [==============================] - 4s 4ms/step - loss: 0.0643 - val_loss: 0.0879\n",
      "Epoch 41/50\n",
      "844/844 [==============================] - 4s 4ms/step - loss: 0.0626 - val_loss: 0.0872\n",
      "Epoch 42/50\n",
      "844/844 [==============================] - 4s 4ms/step - loss: 0.0610 - val_loss: 0.0880\n",
      "Epoch 43/50\n",
      "844/844 [==============================] - 3s 4ms/step - loss: 0.0596 - val_loss: 0.0859\n",
      "Epoch 44/50\n",
      "844/844 [==============================] - 3s 4ms/step - loss: 0.0580 - val_loss: 0.0836\n",
      "Epoch 45/50\n",
      "844/844 [==============================] - 3s 4ms/step - loss: 0.0565 - val_loss: 0.0853\n",
      "Epoch 46/50\n",
      "844/844 [==============================] - 3s 4ms/step - loss: 0.0555 - val_loss: 0.0840\n",
      "Epoch 47/50\n",
      "844/844 [==============================] - 3s 4ms/step - loss: 0.0545 - val_loss: 0.0872\n",
      "Epoch 48/50\n",
      "844/844 [==============================] - 3s 4ms/step - loss: 0.0528 - val_loss: 0.0835\n",
      "Epoch 49/50\n",
      "844/844 [==============================] - 3s 4ms/step - loss: 0.0517 - val_loss: 0.0853\n",
      "Epoch 50/50\n",
      "844/844 [==============================] - 3s 4ms/step - loss: 0.0507 - val_loss: 0.0845\n"
     ]
    }
   ],
   "source": [
    "history = model.fit(X_train_centered, y_train_onehot,\n",
    "                    batch_size=64,\n",
    "                    epochs=50,\n",
    "                    verbose=1,\n",
    "                    validation_split=0.1)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "First 3 predictions:  [5 0 4]\n",
      "First 3 predictions:  [5 0 4]\n",
      "Training accuracy: 98.72%\n",
      "Test accuracy: 97.21%\n"
     ]
    }
   ],
   "source": [
    "#查看准确率，predict_classes函数已经被移除\n",
    "y_train_pred = np.argmax(model.predict(X_train_centered, verbose=0), axis=1)\n",
    "print('First 3 predictions: ', y_train_pred[:3])\n",
    "\n",
    "y_train_pred = np.argmax(model.predict(X_train_centered,\n",
    "                                       verbose=0), axis=1)\n",
    "correct_preds = np.sum(y_train == y_train_pred, axis=0)\n",
    "train_acc = correct_preds / y_train.shape[0]\n",
    "\n",
    "print('First 3 predictions: ', y_train_pred[:3])\n",
    "print('Training accuracy: %.2f%%' % (train_acc * 100))\n",
    "\n",
    "y_test_pred = np.argmax(model.predict(X_test_centered,\n",
    "                                      verbose=0), axis=1)\n",
    "\n",
    "correct_preds = np.sum(y_test == y_test_pred, axis=0)\n",
    "test_acc = correct_preds / y_test.shape[0]\n",
    "print('Test accuracy: %.2f%%' % (test_acc * 100))"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "## 激活函数的选择\n",
    "![](picture/逻辑函数.png)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "P(y=1|x)=0.888\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "X=np.array([1,1.4,2.5])\n",
    "w=np.array([0.4,0.3,0.5])\n",
    "def net_input(X,w):\n",
    "    return np.dot(X,w)\n",
    "\n",
    "def logistic(z):\n",
    "    return 1/(1+np.exp(-z))\n",
    "\n",
    "def logistic_activation(X,w):\n",
    "    z=net_input(X,w)\n",
    "    return logistic(z)\n",
    "\n",
    "print(\"P(y=1|x)=%.3f\"%logistic_activation(X,w))"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Input Z:[1.78 1.36 1.9 ]\n",
      "Output:p: [0.85569687 0.7957597  0.86989153]\n",
      "2\n"
     ]
    }
   ],
   "source": [
    "#使用onehot编码后的输出值\n",
    "W=np.array([[1.1,1.2,0.8,0.4],\n",
    "           [0.2,0.4,1.0,1.2],\n",
    "           [0.6,1.2,0.7,1.5]])\n",
    "A=np.array([[1,0.1,0.4,0.6]])\n",
    "Z=np.dot(W,A[0])\n",
    "y_probs=logistic(Z)\n",
    "print(\"Input Z:%s\"%(Z))\n",
    "print(\"Output:p:\",y_probs)\n",
    "#概率是没有意义的，因为不能理解为多分类的概率值，因为所有概率的和不等于1，但是如果只是用来找到分类的话那是可以的\n",
    "print(np.argmax(y_probs))"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "![](picture/softmax.png)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Softmax p: [0.35912527 0.23596212 0.40491261]\n",
      "1.0000000000000002\n"
     ]
    }
   ],
   "source": [
    "def softmax(z):\n",
    "    return np.exp(z)/np.sum(np.exp(z))\n",
    "y_probs=softmax(Z)\n",
    "print(\"Softmax p:\",y_probs)\n",
    "print(np.sum(y_probs))"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "![](picture/双曲线.png)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "outputs": [
    {
     "data": {
      "text/plain": "<Figure size 432x288 with 1 Axes>",
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAagAAAEYCAYAAAAJeGK1AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAA8fklEQVR4nO3de1xUdf748debOwiK90tqWllqllpq2fVUdqFt7VdshZvtrtXSbtbmftdts4tu0W7sLrsbXexitdSyW9ZGW26FiRmmaIGGihdSEQUVBC8oIDAwn98fZxxBQECBGeD9fDzmMfM+53POvBkH3p5zPufzEWMMSimllLfx8XQCSimlVEO0QCmllPJKWqCUUkp5JS1QSimlvJIWKKWUUl5JC5RSSimv5FUFSkTeEpF9IpLVyHpLREpEJNP1mNveOSqllGoffp5O4AQJwEvAOydp87Ux5pb2SUcppZSneNURlDFmOXDA03kopZTyPG87gmqOySKyDtgDzDbGbGyokYhEA9EA3bp1u3jkyJHtmKJSSqnmWrNmTbExpu+JyztagVoLnGmMKRWRm4H/AiMaamiMeR14HWDChAkmIyOj3ZJUSinVfCKys6HlXnWKrynGmMPGmFLX688AfxHp4+G0lFJKtYEOVaBEZICIiOv1JOz893s2K6WUUm3Bq07xici7gAX0EZF8YB7gD2CMeRX4EfBLEakGjgJRRodjV0qpTsmrCpQxZloT61/C7oaulFKqk+tQp/iUUkp1HVqglFJKeSUtUEoppbySFiillFJeSQuUUkopr6QFSimllFfSAqWUUsoraYFSSinllbRAKaWU8kpaoJRSSnklLVBKKaW8khYopZRSXkkLlFJKKa+kBUoppZRX0gKllFLKK2mBUkop5ZW0QCmllPJKWqCUUkp5JS1QSimlvJIWKKWUUl5JC5RSSimvpAVKKaWUV9ICpZRSyitpgVJKKeWVtEAppZTySlqglFJKeSUtUEoppbySFiillFJeSQuUUkopr+RVBUpE3hKRfSKS1ch6EZEXRGSbiKwXkYvaO0ellFLtw8/TCZwgAXgJeKeR9RHACNfjEuAV17NSSnk1YwzVTkPNsYcxGAMYMBicxm4T4OdDWJB/nW1Ljjooq6zGuPZjjL3c6XrtXg70CPanT2hgne33HDrK4QpHrVxo8DVAv+6B9bbfUVxGWWV1gz/X2X1DCQ7wbcEn0XxedQRljFkOHDhJk1uBd4xtNRAuIgOb2m92djYJCQkAOBwOLMsiMTERgPLycizLYuHChQCUlJRgWRZJSUkAFBcXY1kWixYtAqCgoADLskhOTgYgLy8Py7JISUkBICcnB8uySE1Ndb+3ZVmkpaUBkJWVhWVZpKenA5CZmYllWWRmZgKQnp6OZVlkZdkHkWlpaViWRXZ2NgCpqalYlkVOTg4AKSkpWJZFXl4eAMnJyViWRUFBAQCLFi3CsiyKi4sBSEpKwrIsSkpKAFi4cCGWZVFeXg5AYmIilmXhcNhf5oSEBCzLcn+WCxYsYMqUKe54/vz5REREuOP4+HimTp3qjuPi4oiMjHTHsbGxREVFueOYmBimT5/ujufOncuMGTPc8Zw5c4iOjnbHs2fPZubMme541qxZzJo1yx3PnDmT2bNnu+Po6GjmzJnjjmfMmMHcuXPd8fTp04mJiXHHUVFRxMbGuuPIyEji4uLc8dSpU4mPj3fHERERzJ8/3x1PmTKFBQsWuGPLsvS750XfvbuiojhYVsXO/WXM+n0cP7z313y5pZCPM3cT9cSLXDfzj/z7m11A3e9e2vZiJj/6NhNmJ3Dna6u49eWVXPjoe4x59H2u/ssyLo/9kpGP/oeRv0vigX9mAHW/e6+lbmf4nM8Y8cTnjHwqmfPnLebC33/B2Ke/YOwzXzDumSVcFLOEi59N4fGP7M+/9nfvxaVbuSz2Sy6P/ZIr/rSMK/9sP67+y1dYcV9xTdxXXPvXVK77ayqvLcuGylKmTrmC9994Hg7k8FbSp/wm/h0ej3+Tp194lT+9+CLPv/g3XnnxT/zj5Rj+9fI83n/5CZJefoztSTGUfT6Pt392HltfnQ6Ln2DjGw+w9pX7WP/KT9n8ynS2vjqN3FfvJP/VSKoT7+DAx0+c1nevMd52BNWUM4C8WnG+a9neExuKSDQQDRAYGHjiaqVUJ+KocVJS7UdpyECWbi7kulH966xfsbWYt0tGcvTM8xkfs8S1dBT0G8W9CRmu+CwIg7dW7uDHlwyts/2+w5Xs9ekDPlC8w/V/aJ8wAEr3l7viYAAOljs4ka+PNJK5IYRKulNGdymnB2Wcf2QHrMvmxl75nFeVBsv+yJS8XQz320eIVBBCJSFU0K3W6xCpJJhKAnHgt8YJa+CTK4D8DfDCPJ4EaO6fwRz78dNhQMEiKIBboPFqsQvKjLOZO28ZMSce33mYiAwD/meMGdPAuk+B54wxK1zxUuBRY8yak+1zwoQJJiMj42RNlFIdQHFpJd8XHGFbUSnb95WyvaiM7UWlFByucJ+q6hHsz7p5N9TZLj33AHe8uqpZ7zGwRxCr5lxXZ9nnG/byy3+tbWJLQ3fKuOoM4aWpQ6CsyH6U72fT1u3k7Mylh5QR7ipGYZQRRjn+UtPcH997jbgR7n7/lDcXkTXGmAknLu9oR1D5wJBa8WBgj4dyUUq1k+1FpdzzxjfsKalosm3JUQelldWEBh7/89arW4D7dViQHz2C/QkN9LMfQX50C/QjNMB+3Ts0oN4+LzojmMTb+xFaWUi3ykJCKgoJOlpIYPle/MsK8C0rxPfofsTpgP3AP+puPxoY3TaXaRog4B8MvgHgFwR+ga7nY7FrmW9g3XU+/uDrDz5+rmd/8PWrtfzEuFa7sP5Np3UKOlqB+gR4SETew+4cUWKMqXd6TynV8Tidhsz8Q6zcWsz9V55V58L7oB7BFB6pbHRbEegXFsig8GAGhQdT4aipU6DO7BXC6jnX0bObP4F+DVQKY6C0EA7sgIM74MuF9vOBHXAwl/7lxbTNn2DALxiCwyGoBwQde+4OgWEQ0A38u9nPASEQEAr+Ia441LXM1cY/2H74+NkfSCfgVQVKRN4FLKCPiOQD8wB/AGPMq8BnwM3ANqAcmNHwnpRSHUF1jZNvdhzg86y9LN5YSJGrCI0bGs6VI/q62wUH+DKiXyg7issYObA7I/qFcnbfUM7u241z+oUyuGcIAX6N9/ny8/VhQI8gqHFA0XYo2gJF2VC02X7evx2qj57+DxQQCt36QLe+9iOk9/HX3fpAcK8TilF3+yhGNcirCpQxZloT6w0w82RtlFLeb0dxGQvT8/jPmnyKS+sfGa3ctr9OgQJImDGJPqEB+Pk2s/Nx+QHYu+74o2gLFG8FZ/1ODE0SXwgbCN0HuR5n1H0dNsAuQgEhLd+3apRXFSilVOdljGFZ9j4WLN/Bqpz9Dbbp3S2Aa0b249KzetVbN6BHUOM7rzwC+RmwZy3sybQL0qGdLUswKBx6DYeew6DncNdrV9x9EPi020Uk5aIFSinVLl78cht/W/J9veV9wwK5ecwAbhozkEnDe52kS3YtJbshbzXscj0Ks6C5XZ27nwF9z4O+o1zPI6HPCAipXxSVZ2mBUkq1i9vGn0H80q3UOA0+AteO7EfUxKFY5/Vt+rRd6T7ISYWcr2DHcijZ1fQb+gZAv9EwcCwMGgf9L7ALUlD31vhxVDvQAqWUanU5RaUM6RWCf63CM6RXCHdOGEJIgC/3XTGcQeHBje/AcRRyV0LOMrsoFTY4PGctAv3Ph8ETYdB4uyD1HWV3n1YdlhYopVSrqayu4ZWvtvPysm38+vpzedA6p876526/oPGNS/fB94sh+3PY/uXJe9X5h8AZF8PQyTD0ErswBfVopZ9CeQstUEqpVrF572F+9e53bN1XCkB8ylYixgxkeJ9ujW9UvA02f2IXpfx07GFPG+DjD0MmwVmW/Rg03r5JVHVqWqCUUqfFGEPiN7uI+d8mqqqPd1QYNbA7zoaGUju0CzZ+BBv+AwXrG99xn3PhnOvtgnTmZRAY2vrJK6+mBUopdcrKq6r57Qfr+XTD8QFdgv19efSm8/jJ5GHHe+SVFsHGJMj6EPK+aXhn4mOfsjvvZjgvAnqf3Q4/gfJmWqCUUqekoKSC+95OZ+Oew+5lIweE8dKPL+KcfqFQUw3fL4W178D3yeBsYD4h3wD7KGn0VBhxg3b1VnVogVJKtVjW7hLuezudwsPHR4G4+5KhPHXLaIIO50JKHGS+C6UF9TcWX/u03QU/gpE/0M4NqlFaoJRSLVJZXUP0Oxnu4uTnI8T8cCTTwjfBvx+371NqyOBJMDYKRt9qj0unVBO0QCmlWiTQz5f4aeP5yZvf0tf3CO+M/55hqx+Fkrz6jbv1g3HTYNx06Htu+yerOjQtUEqpFpsYtIevzv2AvrmL8PnuhMFexQfOvQnG3wMjrtfu4OqUaYFSSjXJGIMA5K6Alc/DtpT68yOF9IaLfgoT7oXwIfX2oVRLaYFSSp1U9t4Skv79Kv8X8jmBhd/VbzBwHFzyAJx/O/ifZMRxpVpIC5RSqmE1DkpWv0PQkjjmsAeO1F4pdtfwyQ/Zwwx1khlclXfRAqWUqqvGAevexZn6F3qU7KJ2J3CnbyA+46bBZb/SG2lVm9MCpZSy1VTD+vdg+V/gYC61J8A4bEIoOf8nDIn4Pwird/VJqTahBUqprq6mGja8D6l/hoM76qw6YEJZUH0LQ296mGlXjvFQgqqr0gKlVFdlDGR/Bkvmwv5tdVYdNKEsqP4Bb9fcwG2XnqfFSXmEFiiluqK962DxE5D7dZ3FzqBwXnPczEtl11FGMBOH9WTeD8/3UJKqq9MCpVRXcngvfBkDmf+mztxLgT1wTn6IB7dPJHlrOQA9Q/x5Ydr4OrPiKtWetEAp1RVUlUHaS/ZNto7y48vFFybeD1f/jg82lZG8dYN71d/uHMfAHieZll2pNqYFSqnOzOmE9Qth6TNwZE/ddedGwPXPuMfI+8GFPVifX8K/vtlF9FVncc3Ifh5IWKnjtEAp1VnlroDFj9vXm2rrPwZueBbOvqbO4tBAP/5w2wX8cOwgxg8Nb788lWqEFiilOpv92+2eeVv+V3d5t35w3VMw7m7w8W1080vP6t3GCSrVPFqglOosjh6E1L/At6+D03F8uV8QXPYwXP4IBIbV2aS6xomfdoJQXkoLlFIdXY0D0t+E1Fi7SNV24V1w3VzoMbjeZsYYfpG4hp4hATwWMZLeoYHtlLBSzaMFSqmOyhjI/hyWPFXvRluGToYb/wBnXNzo5ks2FZKyeR8AKZsL+Wr2NfQI0bmblPfwqgIlIjcB8YAv8IYxJvaE9RbwMXBsPJYkY8wz7ZmjUl5h73q7A8QJN9rSc5jdM2/U1JOOMF5eVc3Tiza544gLBmpxUl7HawqUiPgCLwPXA/lAuoh8YozZdELTr40xt7R7gkp5g8N7Ydmz8N2/OPFGW67+LUyKBr+mT9W9+OU2dh86CkCvbgE8euN5bZSwUqfOm66OTgK2GWNyjDFVwHvAra2x4+zsbBISEgBwOBxYlkViYiIA5eXlWJbFwoULASgpKcGyLJKSkgAoLi7GsiwWLVoEQEFBAZZlkZycDEBeXh6WZZGSkgJATk4OlmWRmprqfm/LskhLSwMgKysLy7JIT08HIDMzE8uyyMzMBCA9PR3LssjKygIgLS0Ny7LIzs4GIDU1FcuyyMnJASAlJQXLssjLywMgOTkZy7IoKCgAYNGiRViWRXFxMQBJSUlYlkVJSQkACxcuxLIsysvtmzcTExOxLAuHw77InpCQgGVZ7s9ywYIFTJkyxR3Pnz+fiIgIdxwfH8/UqVPdcVxcHJGRke44NjaWqKgodxwTE8P06dPd8dy5c5kxY4Y7njNnDtHR0e549uzZzJw50x3PmjWLWbNmueOZM2cye/ZsdxwdHc2cOXPc8YwZM5g7d647nj59OjExMe44KiqK2NjjB+6RkZHExcW546lTpxIfH++OIyIimD9/vjueMmUKCxYscMeWZbXOd6+qnCOfPsXRv4yG7xI5VpxqDOQNjIBffUfOgB9gTbmxye/eJ8tW8cbXOe4c7z4/hPCQAP3u6XfPY3/3GuNNBeoMIK9WnO9adqLJIrJORD4XkUYHCRORaBHJEJGMY194pToawTB4/wp48WLC0l8g2NfpXnd0yFXMSB9F9tn3Q7fmdw3/d1YZjhq7wAUe2c01w0NaPW+lWoMYY5pu1Q5E5A7gRmPM/a74HmCSMebhWm26A05jTKmI3AzEG2NGNLXvCRMmmIyMjLZKXam2kbvSdaNtZt3l/c6HG5+Fs69t8S7X7DxA5Cur3PFHD17G+KE9TzNRpU6PiKwxxkw4cbnXXIPCPmIaUiseDNQZm8UYc7jW689EZL6I9DHGFLdTjkq1vf3bIWUebF5Ud3m3fnDtkzB++klvtG2MMYZnP93sju0RI7Q4Ke/lTQUqHRghIsOB3UAU8OPaDURkAFBojDEiMgn7FOX+ds9UqbZw9CAsj4NvXqt/o+3kh+CKWfVutG2JzzYU8N2uQwAE+Ppoxwjl9VpcoESkG1BhjKlpzUSMMdUi8hCwGLub+VvGmI0i8gvX+leBHwG/FJFq4CgQZbzlHKVSp6q6CjLehNQ/1b/R9oI77Rttw4c0vG0LfLGpwP36Z5cPY0gvvfakvFuT16BExAf7aOZuYCJQCQQCRcBnwOvGmK1tnOdp0WtQyisZA5s/gSXz6k21zpBL4cY/wuDGb7Rt+dsZkrMKeHV5Du/MmKT3PSmvcTrXoJYBKcAcIMsY43TtsBdwDRArIh8ZYxJbM2GlOrW8dPjiCcj7pu7y8DPh+qdh9P876Y22p0JEiLhgIBEXDGzV/SrVVppToKYYYxwiEgm4ZzMzxhwAPgQ+FBH9r5hSzXEgB1Kehk3/rbs8KByuftSePLAZN9oq1RU0WaCMMceu1iYC/xWR6ceuP4nIDGPMP2q1UUo1pGw/fP3X+iON+/jDJQ/Alb+BkF6t/raOGie+Ivj4tO7RmFLtoSU36m4BUql7xPTwSdorpSoOw7LnIH4srH65bnE6/zZ4KN0e1LUNihPAWyt2cPMLX7N4YwHan0h1NC3pxWeMMa+KSDnwiYjcDuh/y5RqiOMofLsAVvwdjh6ou27IpfaMtkMmtmkKpZXVvJq6nYPlDh745xpemDaeqWMHtel7KtWaWlKgDgIYY95xFalPAe2nqlRt1VXw3Tv2/UxH9tZd13ekfaPtyFtavQNEQ/65aicHy+0jtsE9g7np/AFt/p5KtaZmFyhjzHW1Xv9HRCqAhLZISqkOp7oK1r9nF6ZDO+uuCz8TrnkcLrjjlEaAOBUVjhreXHF8QNiHrz2HAD9vGnpTqaY1WaBERBq6GdYY8z+gz8naKNXpOSrgu3/CiufhcH7ddaED7Ckwxv8E/ALaNa33M/IoLq0CYGCPIG4bX39GXaW8XbPugxKRD4GPjTG7ji0UkQDgCuCn2PdKJbRJhkp5o6oyyPgHpL0ApYV11wX3hCt+DRN/DgHtfxbcUePktdTjR0/RV52lR0+qQ2pOgboJuBd41zVO3iEgCHs4oi+AvxtjMtsqQaW8SvkBSH8TvnkFyk8YBjKkD1z2kH0v02mMmXe6PsncU2cywqiJQz2Wi1Knozn3QVUA84H5ru7lfYCjxphDbZybUt6jeJvdTTzzXag+Wndd6AC4/BG4+GceOWKqzek0vJK63R3PuGwYwQHtc91LqdbW0sFiBThkjDnaZEulOjpjYGcarHoJsj+nzhTrAD2G2COMj5sO/kGeyLCeJZsL2bavFIDQQD9+MnmYZxNS6jQ0u0CJyCPAXKBCRA4DLxtjXmqzzJTylMpS2PABZLwFBevrrx9wAUx+2L7Rtp07PzTltVpHT3dfOlQHhFUdWnN68T0PrAUeAUYZY/aJSF/gaRGJMcY81cY5KtU+CjfaRWndQqg6Un/9iBvta0zDrmyX+5hOxd/vGsdry3NYtG4P910x3NPpKHVamjPdxm3AeGAWsA84DKzHHjj2F8BEb78epdNtqEZVHrFnrl3zNuStrr/eLwguvAsmz4S+HWeCv/KqakICvGk+UqUad8rTbRhjPgI+EpFLgV8De4GxwIVAL+ArEQk1xpzTyjkr1TacNbBjOax71y5OjvL6bXqPgAn3wrhpdrfxDkaLk+oMWvItngm8D2RiHz2NAjYYYyzXPVFKeS9joDALNvwH1r8PR/bUb+PjB6N+aBcmLz6Np1RX0ZKhjraKyCXA9cA47NN8j7rWVbVJdkqdDmNg7zp77qVNH9tzMTWk7yj7SOnCKAjr364ptoaSow7eSctl2iVD6ROqc0mpzqNF5wFchehT10Mp71NTDfnf2t3CN31cf1y8Y0L62GPjjY2CgWM79NHSe9/u4q9LvufFZdv41bXn8NC1IzydklKtQk9Uq46vrBi2LoGtX8D2pVBR0nC7gDA47yYYEwnnTAHfjt8Fu7rGydtpuQBUVTvpF+Yd92Mp1Rq0QKmOp6oc8r6B3K8h5yvYvZZ6N9EeE9gdzrsZRt8KZ1/rNTfUtpbkjQXsKakAoHe3AKaO0/meVOfRkht1A4FIYFjt7Ywxz7R+WkrV4jgK+emw42u7KOVn1J2Z9kTdz4AR19uF6SwL/DrvdZk3V+xwv7770jMJ8tdhjVTn0ZIjqI+BEmANUNk26aguzxi7M0N+hl2UdmdAwQZwVje+jfjCkEvsojTiBuh/foe+ptRca3cd5LtdhwAI8PXhnkvP9GxCSrWylhSowcaYm9osE9X1OGvgwA4o3AAFWXaPu90ZcPRg09v2HWl3BR9+JQy/qkPeq3S63qp19DR13CD6hnXeI0XVNbWkQKWJyAXGmA1tlo3qnIyBIwWwfysUZdtHRIVZsG9zwzfJNqT3CBh2hV2Qhl0Jof3aNmcvt/vQUT7PKnDH916uwxqpzqclBeoK4GcisgP7FJ8AxhhzYZtkpjoWY6DiEBzaBfu32dNT7N8KxVth//aGx7ZrTFA4DJ4Agyfaz2dc3CWPkE7mnbRcapx2x5DJZ/Vm9KDuHs5IqdbXkgIV0WZZKO9XUw1l++DIXjiUZxeiEtfzsbglReiYbv1gwBjoP8YeJXzQRdD77C5xDelUlVVW8+637smtdVBY1Wm1ZCSJnSIyFrjStehrY8y6tklLtYvqSvt6T/kBOHoAyorgSKE9hXlpoX1a7thz+X4a7crdHIE9oM859qm6/qOPF6QufqruVOzcX06PEH8OV1QzrHcI147Uz1B1Ti2dD+rnQJJrUaKIvG6MebFNMlMnZ4x9/aay1B6Ru+qI/VxZClWlUHn4+LqjB+0CdKwQHT1kv3aUtW5O/iH2JH69hkPvc6DPCLsg9RkB3frqUVErGT2oO1/NvoYlmwoRAR8f/VxV59SSU3z3AZcYY8oARORPwCqg1QqUiNwExAO+wBvGmNgT1otr/c1AOfAzY8za1nr/ZnM67ftwahxQU2V3gW70dZUdu9s77COX6qPgqGj587GiVHUEjLN9f+6QPhA2wC5C4UMgfKjr9VD7EdJbi1A78fURbhozwNNpKNWmWlKgBKipFde4lrUKEfEFXsYejDYfSBeRT4wxm2o1iwBGuB6XAK+4ntvG8jj4dkH9YmNqmt62I/DxszsfBPeyn0N624Olhg44/hzazy5K3fp2iqGBlFIdh08L2v4D+EZEfi8ivwdWA2+2Yi6TgG3GmBzXoLTvAbee0OZW4B1jWw2Ei8jApnacnZ1NQkICAA6HA8uySExMBKC8vBzLsli4cCEAJSUlWJZFUlKSfbRSWmCfFqs8DNUV3lWc/IKp9O9BfnkgVb1HwZmXsy/8IlIKe1I++i647GG2DIwkfutgDt/wd5iexJfnzCVq9WhKHtwITxWzcOgfsP4TQvm0j2Dav0k8dDHW7z/HceHdcN5NJCxZjzX1x+7itGDBAqZMmeJOYf78+UREHO8/Ex8fz9SpU91xXFwckZGR7jg2NpaoqCh3HBMTw/Tp093x3LlzmTFjhjueM2cO0dHR7nj27NnMnDnTHc+aNYtZs2a545kzZzJ79mx3HB0dzZw5c9zxjBkzmDt3rjuePn06MTEx7jgqKorY2OMH7pGRkcTFxbnjqVOnEh8f744jIiKYP3++O54yZQoLFixwx5Zlndp3DyguLsayLBYtWgRA3u49WJZFcnKyHeflYVkWKSkpAOTk5GBZFqmpqYD9vbcsi7S0NACysrKwLIv09HQAMjMzsSyLzMxMANLT07Esi6ysLADS0tKwLIvs7GwAUlNTsSyLnBx7VPiUlBQsyyIvLw+A5ORkLMuioMDu/r5o0SIsy6K4uBiApKQkLMuipMQeK3HhwoVYlkV5uX2rQWJiIpZl4XDYo4QkJCRgWZb7s9Tvnue+ewUFBW363WtMswuUMeZvwL3AAeAgMMMY83xzt2+GM4C8WnG+a1lL2wAgItEikiEiGce+8C3m0/gRg1N8wT8EZ0B3Dlb5URHQC8KHUhU2lB1lQRwJHQ6DLqK8z1i+OxjKod7jYcSNHB50FV/uC6do8I0w8X72nXMX7+T2Z8+o++HGP5I75hGe2zyUXZc8A9PeY+PFf+ShtSPIvfEdmPktqye/wS0rLiBnejo8WcDXl7zF9G9HU/j/PoAZn7F25O94dvMwDl/1DNzwLFsH3cZHu/tSde5UOOc6DoUMp6AiEALD9HRcB/OnpbvYO3oa3xUZdxdzpTqzJqd8by8icgdwozHmfld8DzDJGPNwrTafAs8ZY1a44qXAo8aYNSfb9ylP+V5RAlVl4Btgnw7zDbCPJHz89I+7aldFRyq5PPZLqmrs644fPXgZ44fqvWGqczjlKd9FZIUx5goROULdfsbHbtRtrTsE84EhteLBwInTnjanTesJ6mE/lPKwf32z012cxg0J1+KkuoQmT/EZY65wPYcZY7rXeoS1YnECSAdGiMhw1xTyUcAnJ7T5BPiJ2C4FSowxe1sxB6W8ToWjhsTVxyde1BtzVVfR7GtQrm7lTS47VcaYauAhYDGwGXjfGLNRRH4hIr9wNfsMyAG2AQuAB1vr/ZXyVp+s20NxaRUAA3sEafdy1WW0pJv59cDvTlgW0cCyU2aM+Qy7CNVe9mqt1waYeeJ2SnVWxpg6o5b/9LJh+Pu2pPOtUh1Xc65B/RL7SOUsEVlfa1UYkNZWiSmlIG37frYU2GMcBvv7Mm3iUA9npFT7ac4R1L+Bz4HngMdqLT9ijDnQJlkppQB44+sc9+s7JwymR4jeLK26jiYLlDGmBHsm3Wki0hN7FIcgABHBGLO8bVNUqmvatq+UZdlFgH1Xwwyd80l1MS0ZLPZ+4BHsrt2ZwKXYY/Fd2yaZKdXF/WPl8WtP143sz7A+3TyYjVLtryVXWx8BJgI7jTHXAOOBojbJSinFTyYP464JQwjw8+H+K/XoSXU9LenFV2GMqRARRCTQGLNFRM5rs8yU6uLOGxDGn350Ib+LGElPvfakuqCWFKh8EQkH/gssEZGDtOUoDkopAHp1C/B0Ckp5REtm1L3N9fL3IrIM6AEkt0lWSimluryWjCTxaxEZDGCMSTXGfOKaFkMp1UqMMfz7m10cKtdfLaVa0kmiO7BYRL4WkZki0r+tklKqq1qdc4DHP9rApc8t5dn/bWp6A6U6sZbMB/W0MeZ87KGGBgGpIpLSZpkp1QW9tnw7ABUOJ2VVXjQ5plIecCqDeu0DCoD9QL/WTUeprmvTnsN8VevG3OirzvJwRkp5VkuuQf1SRL4ClgJ9gJ8bYy5sq8SU6mqOHT0BRIwZwHC9MVd1cS3pZn4mMMsYk9lGuSjVZeUdKGfRuuN3bfzi6rM9mI1S3qEl3cwfa7qVUupULPg6B6drvurLz+nNhYPDPZqPUt7Am6Z8V6pLKi6tZGF6njv+5dXneDAbpbxHc0Yzd0/53vbpKNX1vJ2WS2W1E4AxZ3Tn8nN6ezgjpbyD10z5rlRXVFpZzTurdrrjX159DiLiwYyU8h4t6WZ+fQPLIlorEaW6orwD5YS7BoI9s3cIN40Z4OGMlPIeLZny/Wyd8l2p1jVqYHeW/t/V/DdzD6GBfvj66NGTUsfolO9KeZifrw8/uniwp9NQyus0eYrPGFNijMkFqoASY8xOY8xOwIjIW22doFJKqa6pJdegLjTGHDoWGGMOYs+qq5RqoT2HjmKMabqhUl1YSwqUj4j0PBaISC9aNhKFUgq7594PXviaO19bxcptxVqolGpESwrUX4E0EYkRkRjsDhJ/bpu0lOq8Elbu4GC5g/Tcg/zuw/VUO7VAKdWQlgx19I6IZADXYo8icbsxRiesUaoFDpRV8Wpqjjt++Npz8Pc9lUkFlOr8WvqbsRf4FlgH9BGRq1o/JaU6r5e+3EZpZTUAZ/ftRuRF2ntPqcY0+whKRO4HHgEGA5nApcAq7CMqpVQT8g6U88/Vue740ZtG4qdHT0o1qiW/HY8AE4GdxphrsHvwFbVJVkp1Qn9b8j2OGvt600VDw7lhdH8PZ6SUd2tJL7wKY0yFiCAigcaYLSJyXmsk4eoRuBAYBuQCd7q6sZ/YLhc4AtQA1caYCa3x/kq1tazdJfw3c7c7nnPzKB1zT6kmtOQIKl9EwoH/AktE5GNgz0m3aL7HgKXGmBHYM/aebO6pa4wx47Q4qY7CGMMzizZxrDf5lFH9mTisl2eTUqoDaHaBMsbcZow5ZIz5PfAU8Cbw/1opj1uBt12v327F/QKQnZ1NQkICAA6HA8uySExMBKC8vBzLsli4cCEAJSUlWJZFUlISAMXFxViWxaJFiwAoKCjAsiySk5MByMvLw7IsUlJSAMjJycGyLFJTU93vbVkWaWn2sIVZWVlYlkV6ejoAmZmZWJZFZmYmAOnp6ViWRVZWFgBpaWlYlkV2djYAqampWJZFTo7dEywlJQXLssjLs+cTSk5OxrIsCgoKAFi0aBGWZVFcXAxAUlISlmVRUlICwMKFC7Esi/LycgASExOxLAuHwwFAQkIClmW5P8sFCxYwZcoUdzx//nwiIo6PGRwfH8/UqVPdcVxcHJGRke44NjaWqKgodxwTE8P06dPd8dy5c5kxY4Y7njNnDtHR0e549uzZzJw50x3PmjWLWbNmueOZM2cye/ZsdxwdHc2cOXPc8YwZM5g7d647nj59OjExMe44KiqK2NhYdxwZGUlcXJw7njp1KvHx8e44IiKC+fPnu+MpU6awYMECd2xZFrNffI9vc12jgjlrOL/6e0C/e/rda/vvXkf5u9eYU7pCa4xJNcZ8YoypOpXtG9DfGLPXte+9QL/G3hr4QkTWiEh0I20AEJFoEckQkYxjX3ilPOGQwwc/1yCw3QvW0jfQ6eGMlOoYpL3uYheRFKChuQSeAN42xoTXanvQGNPzxIYiMsgYs0dE+gFLgIeNMcubeu8JEyaYjIyMU09eqdO0tfAIL3y5jT/cNobuQf6eTkcpryIiaxq6bNNuQxUZY6Y0tk5ECkVkoDFmr4gMBPY1so89rud9IvIRMAloskAp5Wkj+ofx4jQdulKplvCWmzA+AX7qev1T4OMTG4hINxEJO/YauAHIarcMlVJKtStvKVCxwPUishV75t5YsE/pichnrjb9gRUisg57NItPjTHJHslWqSYsy95H3oFyT6ehVIfmFaORG2P2A9c1sHwPcLPrdQ4wtp1TU6rFCg9X8Kt3v6O6xvCbG85lxuXDdaZcpU6BtxxBKdUpGGN46r9ZHKmo5qijhsTVO3HUaK89pU6FFiilWtEHa/L5YlOhO/7j7RcQ5O/rwYyU6ri0QCnVSnKLy3j6k43u+O5LhnLZ2X08mJFSHZsWKKVagaPGyayFmZRV1QBwVt9uPPmD0R7OSqmOTQuUUq3gb0u+JzPvEAB+PkL8XeMJDtBTe0qdDi1QSp2m5KwCXvlquzv+vxvO5YLBPTyYkVKdgxYopU7Dtn2lzP5gnTu++ty+PHDV2R7MSKnOQwuUUqchM+8Q5VX2FO5DegUTHzVO73lSqpV4xY26SnVUP7p4MIPCg/jtB+t55e6LCQ8J8HRKSnUaWqCUOk2Xnd2HZbMtAvz0hIRSrUl/o5RqAWMMhyvqzy+mxUmp1qe/VUq1wPMpW7nlhRXsPnTU06ko1elpgVKqmV5YupX4pVvZdaCcu15bxd4SLVJKtSW9BqVUE4wx/GVxNvNr3et0dt9QemqHCKXalBYopU7CUePksQ838OHafPeyK0f04bV7LtZBYJVqY1qglGpESbmDh9/7juXfF7mXXTeyHy/9+CItTkq1Ay1QSjVg897DPPDPNeyqNSvuXROG8IfbxuDnq5dulWoPWqCUOsF/1uTz5H83UOE4PtHgr64bwa+njEBER4lQqr1ogVLqBMWlle7i1C3Al7/eOZabxgz0cFZKdT1aoJQ6wc+vPIulmws5UFbFa/dczDn9wjydklJdkhYo1aVl7S5BBM4fdHx6DF8f4eUfX0RYkL/O6aSUB2mBUl3Stn1HeD5lK59u2MuYQT3478zL64xC3q97kAezU0qBFijVxazLO8SbK3awaP0ejLGXbdhdwnvpu7j7kjM9m5xSqg4tUKrTq3DUsHhjAW+n5bJ216F666eM6s/ks3q3f2JKqZPSAqU6rcy8Q/z7m518vqGAI5XV9dZfc15ffn39uVw4OLz9k1NKNUkLlOq0MnIP8H5Gfp1lAb4+3DJ2ID+7bJgWJqW8nBYo1WHtLTlKRu5B1uw8SNbuEt6LvrTOKA8/HDuIP3y2GWPgzN4hRF40mGmThtI3LNCDWSulmksLlPJ6Ryoc7NxfzpaCI2zZe9h+LjhMcWlVnXZbCo4w5ozj3cX7dw9i3i2jGTsknHFDwnUUCKU6GC1QyqPKq6opPlJFUWkF/cKCGNIrpM762+av5LsGOjY0ZM3Og3UKFMDPLh/eWqkqpdqZVxQoEbkD+D0wCphkjMlopN1NQDzgC7xhjIlttyQVxhgqq51UOpxUVNdQ4aihstpJhaOGCoeT8BB/zu1fd9SFJZsKWZ2zn8NHHRypqOZwhf1cctTB/tJKyqpq3G1/e+N5zLzmnDrb9wj2bzSfYH9fxg7pwcRhvbj4zJ5cdGbP1v2BlVIe5RUFCsgCbgdea6yBiPgCLwPXA/lAuoh8YozZ1JaJVdc4eeKjLAzGfd+MAYwBw/EFrlc8c+v5hAUd/6N6oKyKeZ9sxBh3a1f7Wvtzxb4+wvy7L67z/tuLSnnOdR3Ffl9TJ4fjrw29uwXwfNT4Ott/u+MAz6d8j9MYnE6odjqpcRqqnYaaWo9qp+H8Qd15ZXrd9//nqlye/XSzu83J3HLhQF768UV1lqVtL+YfK3NPut0xRUcq6y0b2iuEAD8fhvQMZkS/MM4bEMaogWGMHNCdob1C8PHR03ZKdVZeMW+AMWazMSa7iWaTgG3GmBxjTBXwHnBrc/afnZ1NQkICAA6HA8uySExMBKC8vBzLsli4cCEAJSUlWJZFUlISAMXFxSzMyOP9jHw+WGM//rMmnw/X5pO0drf9+G43H7ke32/fgWVZpKamArBxy/csWreH/63fy6fHHhv28tmGAj7Psh/JGwtYvLGQxRsLSU9Px7IssrKyAFjxzVpSNu9j6ZZ9fLllH8uyi1iWXcRX2UWkfm8/ln9fxNdbi/lmxwGSk5OxLIuCggIAFqeuJG37flbnHODb3AOs3XWIdfklbNxjX8vZuq+UnOIydh0oZ29JBYmJiViWhcPhAGDV6tVUVjubLE4AldVO4uPjmTp1qntZ1tr0k27jg5PA6jLGDu7BgB5BzJ07lxkzZrjXH131Llfv+4ilv7F49Z6L2b34dRa98izD+nTDx0eYNWsWs2bNcrefOXMms2fPdsfR0dHMmTPHHc+YMYO5c+e64+nTpxMTE+OOo6KiiI09fmAeGRlJXFycO546dSrx8fHuOCIigvnz57vjKVOmsGDBAndsWdZpffcsy2LRokUAFBQUYFkWycnJAOTl5WFZFikpKQDk5OTU+e5lZ2djWRZpaWkAZGVlYVkW6en2v0lmZiaWZZGZmQlQ77uXlpaGZVlkZ9u/mqmpqViWRU5ODgApKSlYlkVeXh5Ave/eokWLsCyL4uJiAJKSkrAsi5KSEgAWLlyIZVmUl9tTmpz43UtISMCyLPdnuWDBAqZMmeKO58+fT0REhDs+8bsXFxdHZGSkO46NjSUqKsodx8TEMH36dHd84ndvzpw5REdHu+PZs2czc+ZMd6zfvdb77jXGW46gmuMMIK9WnA9c0lhjEYkGogECA0+j15aHL6yf7tu3ZHOnqV+ETtzeVwzGUUnvnj0I8veh/HAJZYcPMWbUeZzdNxQO1m0/NKCMvbt28NAD9xEW5M9nH3/Izq2bif/rn+jTLZD4uFi+/z6bxDj7F2fu0rrb+4mzRT+DUp7mcDjIz8/nnnvuwdfXl82bNwNw77334ufn544feOCBOvGDDz5IQECAO37kkUcIDAx0x7/5zW8ICgpyx4899hjBwcHu+MknnyQkJMQdz5s3j9DQUDZv3owxpk7sdDqZN28eYWFhdeLu3buzefNmampqGox79OjB5s2bqa6uZt68eYSHh9eJj7V3OBzMmzfPnU9NTQ1z587Fz8/P/R+Q5hDTwB+ltiAiKcCABlY9YYz52NXmK2B2Q9egXNepbjTG3O+K78G+XvVwU+89YcIEk5HR4GWtJjmdhvcz8lw5gBz7cyn2H+9jPcPEtf7mCwbWmW31aFUNX2wqOPYzuP/YHtuXHN8dIsJNY+p+RCXlDr7NPVDnPY5vI9RKh0A/XyafXXdEhOLSSrILjiDYg6D6+Qq+Pj74itSKBV8Rgvx9GdCj7hh0jhr7lKCfj91Oe8IpdXI7duwgLCyM3r176+9LLcYY9u/fz5EjRxg+vG7nJRFZY4yZcOI27XYEZYyZ0nSrk8oHhtSKBwN7TnOfTfLxEaImDT3l7YMDfLl13BmnvH2PEH+uH93/lLfvExpIn3NO/QjS39cHnd1cqearqKhg2LBhWpxOICL07t2boqKiZm/jFdegmikdGCEiw0UkAIgCPvFwTkopVY8Wp4a19HPxigIlIreJSD4wGfhURBa7lg8Skc8AjDHVwEPAYmAz8L4xZqOnclZKKdW2vKJAGWM+MsYMNsYEGmP6G2NudC3fY4y5uVa7z4wx5xpjzjbG/MFzGSullHc6dOhQnd59LWVZFqd6zb61eUWBUkop1TpOt0B5k47UzVwppTqcvy/5nvilW5vVdtqkITx3+4V1ls1JWs+73x6/w+aR60bw6+vPbXQfjz32GNu3b2fcuHFcc801rF+/noMHD+JwOHj22We59dZbyc3NJSIigiuuuIK0tDTOOOMMPv74Y4KDgwH44IMPePDBBzl06BBvvvkmV1555Sn85KdPC5RSSnUisbGxZGVlkZmZSXV1NeXl5XTv3p3i4mIuvfRS983MW7du5d1332XBggXceeedfPjhh+4bl6urq/n222/57LPPePrpp9035LY3LVBKKdVJGWN4/PHHWb58OT4+PuzevZvCwkIAhg8fzrhx4wC4+OKLyc3NdW93++23N7i8vWmBUkqpNvTr68896Sm5pjx3+4X1Tvs117/+9S+KiopYs2YN/v7+DBs2jIqKCqDuCDu+vr4cPXrUHR9b5+vrS3V1/dmo24t2klBKqU4kLCyMI0eOAPYYe/369cPf359ly5axc+dOD2fXMnoEpZRSnUjv3r25/PLLGTNmDBMnTmTLli1MmDCBcePGMXLkSE+n1yLtNhafJ53OWHxKKdUSmzdvZtSoUZ5Ow2s19Pk0NhafnuJTSinllbRAKaWU8kpaoJRSSnklLVBKKaW8khYopZRSXkkLlFJKKa+kBUoppTqZ0NDQU972/vvvZ9OmTY2uT0hIYM+ePc1ufzr0Rl2llFJub7zxxknXJyQkMGbMGAYNGtSs9qdDC5RSSrWV3/dow32XNNnEGMOjjz7K559/jojw5JNPctddd+F0OnnooYdITU1l+PDhOJ1O7r33Xn70ox9hWRZxcXGMHz+e++67j4yMDESEe++9lyFDhpCRkcHdd99NcHAwq1atIiIigri4OCZMmEBycjKPP/44NTU19OnTh6VLl57Wj6gFSimlOqmkpCQyMzNZt24dxcXFTJw4kauuuoqVK1eSm5vLhg0b2LdvH6NGjeLee++ts21mZia7d+8mKysLsCdCDA8P56WXXnIXpNqKior4+c9/zvLlyxk+fDgHDhw47fz1GpRSSnVSK1asYNq0afj6+tK/f3+uvvpq0tPTWbFiBXfccQc+Pj4MGDCAa665pt62Z511Fjk5OTz88MMkJyfTvXv3k77X6tWrueqqqxg+fDgAvXr1Ou389QhKKaXaSjNOw7WlxsZabc4YrD179mTdunUsXryYl19+mffff5+33nrrpO8lIqeca0P0CEoppTqpq666ioULF1JTU0NRURHLly9n0qRJXHHFFXz44Yc4nU4KCwv56quv6m1bXFyM0+kkMjKSmJgY1q5dC9SdzqO2yZMnk5qayo4dOwBa5RSfHkEppVQnddttt7Fq1SrGjh2LiPDnP/+ZAQMGEBkZydKlSxkzZgznnnsul1xyCT161O3QsXv3bmbMmIHT6QTgueeeA+BnP/sZv/jFL9ydJI7p27cvr7/+OrfffjtOp5N+/fqxZMmS08pfp9tQSqlW1FGm2ygtLSU0NJT9+/czadIkVq5cyYABA9r8fVsy3YYeQSmlVBd0yy23cOjQIaqqqnjqqafapTi1lBYopZTqghq67uRttJOEUkq1sq5w6eRUtPRz0QKllFKtKCgoiP3792uROoExhv379xMUFNTsbfQUn1JKtaLBgweTn59PUVGRp1PxOkFBQQwePLjZ7bVAKaVUK/L393ePpqBOj1ec4hORO0Rko4g4RaReV8Na7XJFZIOIZIqI9htXSqlOzFuOoLKA24HXmtH2GmNMcRvno5RSysO8okAZYzYDrT6Ok1JKqY7LKwpUCxjgCxExwGvGmNcbaygi0UC0KywVkez2SLAN9QG6+pGjfgY2/Rxs+jnYOsPncGZDC9utQIlICtDQrcpPGGM+buZuLjfG7BGRfsASEdlijFneUENX8Wq0gHU0IpLR0FAgXYl+Bjb9HGz6Odg68+fQbgXKGDOlFfaxx/W8T0Q+AiYBDRYopZRSHZtX9OJrDhHpJiJhx14DN2B3rlBKKdUJeUWBEpHbRCQfmAx8KiKLXcsHichnrmb9gRUisg74FvjUGJPsmYw9otOcrjwN+hnY9HOw6edg67SfQ5eYbkMppVTH4xVHUEoppdSJtEAppZTySlqgOiARmS0iRkT6eDqX9iYifxGRLSKyXkQ+EpFwT+fUnkTkJhHJFpFtIvKYp/NpbyIyRESWichm1/Boj3g6J08SEV8R+U5E/ufpXNqCFqgORkSGANcDuzydi4csAcYYYy4EvgfmeDifdiMivsDLQAQwGpgmIqM9m1W7qwZ+Y4wZBVwKzOyCn0FtjwCbPZ1EW9EC1fH8HXgUe1SNLscY84UxptoVrgaaP3Z/xzcJ2GaMyTHGVAHvAbd6OKd2ZYzZa4xZ63p9BPuP8xmezcozRGQw8APgDU/n0la0QHUgIjIV2G2MWefpXLzEvcDnnk6iHZ0B5NWK8+mif5wBRGQYMB74xsOpeMrz2P9ZdXo4jzbT0cbi6/RONiQU8Dj2DcqdWnOGxRKRJ7BP9/yrPXPzsIZGU+6SR9IiEgp8CMwyxhz2dD7tTURuAfYZY9aIiOXhdNqMFigv09iQUCJyATAcWOca9X0wsFZEJhljCtoxxTbX1LBYIvJT4BbgOtO1buTLB4bUigcDezyUi8eIiD92cfqXMSbJ0/l4yOXAVBG5GQgCuotIojFmuofzalV6o24HJSK5wISuNjeWiNwE/A242hjTpebUFhE/7I4h1wG7gXTgx8aYjR5NrB2J/b+zt4EDxphZHk7HK7iOoGYbY27xcCqtTq9BqY7mJSAMezT7TBF51dMJtRdX55CHgMXYnQPe70rFyeVy4B7gWte/f6brKEJ1QnoEpZRSyivpEZRSSimvpAVKKaWUV9ICpZRSyitpgVJKKeWVtEAppZTySlqglFJKeSUtUEoppbySFiilPEBEwkXkwZOsT2vv91TK22iBUsozwoFGi4Ux5rL2fk+lvI0WKKVagYgMc83yusA10+sXIhLsWjddRL51DcvzmmviwVjgbNeyvzSwv9KT7de1fIuIvO2aXfg/IhJSa5usWvuaLSK/b8Z7fllr+KAKEbmjTT4spZpJC5RSrWcE8LIx5nzgEBApIqOAu4DLjTHjgBrgbuAxYLsxZpwx5rct3a9r+XnA667ZhQ/T9NHRSd/TGHOtK8fXgE+ArjpSuPISWqCUaj07jDGZrtdrgGHYI49fDKSLSKYrPqsV9guQZ4xZ6XqdCFxxKknXJiI/wZ5S/m5jTM3p7k+p06HzQSnVeiprva4BgrEnGXzbGDOndkPXbLCns1+oP1nhsbiauv/5DGrOm7hO6d0N3GqMcbQgP6XahB5BKdW2lgI/EpF+ACLSS0TOBI5gTxtyOoaKyGTX62nACtfrQqCfiPQWkUDsyR052Xu6Zmh9ELjdGFNxmnkp1Sq0QCnVhowxm4AngS9EZD2wBBhojNkPrBSRrIY6LDTTZuCnrv32Al5xvacDeAb4BvgfsMW1/GTv+Tb2DL0rXZ0k7jvFnJRqNToflFIdkOsU4f+MMWM8nYtSbUWPoJRSSnklPYJSSinllfQISimllFfSAqWUUsoraYFSSinllbRAKaWU8kpaoJRSSnklLVBKKaW8khYopZRSXun/A6huyOvHI3a1AAAAAElFTkSuQmCC\n"
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "def tanh(z):\n",
    "    e_p=np.exp(z)\n",
    "    e_m=np.exp(-z)\n",
    "    return (e_p-e_m)/(e_p+e_m)\n",
    "import matplotlib.pyplot as plt\n",
    "z = np.arange(-5, 5, 0.005)\n",
    "log_act = logistic(z)\n",
    "tanh_act = tanh(z)\n",
    "\n",
    "plt.ylim([-1.5, 1.5])\n",
    "plt.xlabel('net input $z$')\n",
    "plt.ylabel('activation $\\phi(z)$')\n",
    "plt.axhline(1, color='black', linestyle=':')#增加水平线\n",
    "plt.axhline(0.5, color='black', linestyle=':')\n",
    "plt.axhline(0, color='black', linestyle=':')\n",
    "plt.axhline(-0.5, color='black', linestyle=':')\n",
    "plt.axhline(-1, color='black', linestyle=':')\n",
    "\n",
    "plt.plot(z, tanh_act,\n",
    "         linewidth=3, linestyle='--',\n",
    "         label='tanh')\n",
    "\n",
    "plt.plot(z, log_act,\n",
    "         linewidth=3,\n",
    "         label='logistic')\n",
    "plt.legend(loc='lower right')\n",
    "plt.tight_layout()\n",
    "#plt.savefig('images/13_03.png')\n",
    "plt.show()\n",
    "#np.tanh#现成的双曲正切函数\n",
    "#from scipy import expit#现成的logi函数"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "![](picture/relu.png)\n",
    "![](picture/激活函数.png)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "# 总结\n",
    "## 这一章学会了如何使用开源的TensorFlow在深度学习中进行数值计算。虽然因为TensorFlow支持GPU所带来的额外复杂性使其用起来不如NumPy方便，但是它可以非常有效地定义和训练大型多层神经网络。同样你也学习了如何调用TensorFlow的API来构建复杂的机器学习和神经网络模型并有效地运行。我们首先探讨了TensorFlow的低级API编程。当为矩阵向量乘法编程并定义每个操作的细节时，调用低级API建模可能会很麻麻烦。然而好处是它允许研发人员组合这些基本操作以构建更复杂的模型。此外也讨论了TensorFlow如何利用GPU来加快大型神经网络训练和测试的计然后探索了两个高级API，用它们构建神经网络模型要比低级API容易得多。具体来说，使用TensorFlow的Layers和Keras建立多层神经网络，并学习如何使用这些API来建立模型。tanh、softmax、和ReLU。第12章从实现简单的多层感知器（MLP）识别MNIST手写图像数据集开始。而从零开始的底层实现有助于说明多层神经网络的核心概念，如正向和反向传播的神经网络，用NumPy训练神经网络非常低效而且不切合实际。"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}